Merge master

README.md

@@ -9,7 +9,7 @@
 [![Badge](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu/#/en_US)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/javiercp/BinderTF.NET/master?urlpath=lab)
 
-*master branch is based on tensorflow 2.3 now, v0.15-tensorflow1.15 is from tensorflow1.15.*
+*master branch is based on tensorflow v2.4, v0.3x branch is based on tensorflow v2.3, v0.15-tensorflow1.15 is from tensorflow1.15.*
 
 
 ![tensors_flowing](docs/assets/tensors_flowing.gif)
@@ -30,7 +30,8 @@ Go through the online docs [TensorFlow for .NET](https://scisharp.github.io/tens
 
 | TensorFlow                 | tf native1.14, cuda 10.0 | tf native 1.15, cuda 10.0 | tf native 2.3, cuda 10.1 | tf native 2.4, cuda 11 |
 | -------------------------- | ------------- | -------------- | ------------- | ------------- |
-| tf.net 0.3x, tf.keras 0.2 |               |                | x | not compatible |
+| tf.net 0.4x, tf.keras 0.5 |  |  |  | x |
+| tf.net 0.3x, tf.keras 0.4 |               |                | x |  |
 | tf.net 0.2x          |               | x | x |              |
 | tf.net 0.15          | x | x |               |               |
 | tf.net 0.14          | x |                |               |               |
@@ -50,10 +51,10 @@ PM> Install-Package TensorFlow.Keras
 
 ### Install tensorflow binary
 ### For CPU version
-PM> Install-Package SciSharp.TensorFlow.Redist -Version 2.3.1
+PM> Install-Package SciSharp.TensorFlow.Redist
 
 ### For GPU version (CUDA and cuDNN are required)
-PM> Install-Package SciSharp.TensorFlow.Redist-Windows-GPU -Version 2.3.1
+PM> Install-Package SciSharp.TensorFlow.Redist-Windows-GPU
 ```
 
 Import TF.NET and Keras API in your project.

TensorFlow.NET.sln

@@ -7,7 +7,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Binding", "src\T
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Benchmark", "src\TensorFlowNet.Benchmarks\Tensorflow.Benchmark.csproj", "{3A6EB896-604F-4E25-B677-B8103BCF3D2E}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.UnitTest", "test\TensorFlowNET.UnitTest\Tensorflow.UnitTest.csproj", "{23C28035-2FCE-41F3-9A12-E73CE8A5AE32}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Binding.UnitTest", "test\TensorFlowNET.UnitTest\Tensorflow.Binding.UnitTest.csproj", "{23C28035-2FCE-41F3-9A12-E73CE8A5AE32}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Console", "src\TensorFlowNET.Console\Tensorflow.Console.csproj", "{03F06299-3F4B-4449-A709-3A647657BC0C}"
 EndProject

src/TensorFlowNET.Console/MemoryBasicTest.cs

@@ -4,6 +4,8 @@ using Tensorflow.Keras.ArgsDefinition;
 using Tensorflow.Keras.Engine.DataAdapters;
 using static Tensorflow.Binding;
 using static Tensorflow.KerasApi;
+using System.Linq;
+using System.Collections.Generic;
 
 namespace Tensorflow
 {
@@ -35,13 +37,15 @@ namespace Tensorflow
         public Action<int, int> ConstantString
             => (epoch, iterate) =>
             {
-                var tensor = tf.constant(new string[] 
+                var strList = new string[]
                 {
                     "Biden immigration bill would put millions of illegal immigrants on 8-year fast-track to citizenship",
                     "The Associated Press, which also reported that the eight-year path is in the bill.",
                     "The bill would also include provisions to stem the flow of migration by addressing root causes of migration from south of the border."
-                });
-                var data = tensor.numpy();
+                };
+
+                var tensor = tf.constant(strList, TF_DataType.TF_STRING);
+                var data = tensor.StringData();
             };
 
         public Action<int, int> Variable
@@ -108,16 +112,18 @@ namespace Tensorflow
                 var strides = new[] { 1, 1, 1, 1 };
                 var dilations = new[] { 1, 1, 1, 1 };
 
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Conv2D", null,
-                    null,
-                    input, filter,
-                    "strides", strides,
-                    "use_cudnn_on_gpu", true,
-                    "padding", "VALID",
-                    "explicit_paddings", new int[0],
-                    "data_format", "NHWC",
-                    "dilations", dilations);
+                var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("Conv2D", null, input, filter)
+                {
+                    attrs = ConvertToDict(new
+                    {
+                        strides,
+                        use_cudnn_on_gpu = true,
+                        padding = "VALID",
+                        explicit_paddings = new int[0],
+                        data_format = "NHWC",
+                        dilations
+                    })
+                });
             };
 
         public Action<int, int> Conv2DWithVariable
@@ -128,16 +134,18 @@ namespace Tensorflow
                 var strides = new[] { 1, 1, 1, 1 };
                 var dilations = new[] { 1, 1, 1, 1 };
 
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Conv2D", null,
-                    null,
-                    input, filter,
-                    "strides", strides,
-                    "use_cudnn_on_gpu", true,
-                    "padding", "VALID",
-                    "explicit_paddings", new int[0],
-                    "data_format", "NHWC",
-                    "dilations", dilations);
+                var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("Conv2D", null, input, filter)
+                {
+                    attrs = ConvertToDict(new
+                    {
+                        strides,
+                        use_cudnn_on_gpu = true,
+                        padding = "VALID",
+                        explicit_paddings = new int[0],
+                        data_format = "NHWC",
+                        dilations
+                    })
+                });
             };
 
         public Action<int, int> Dataset

src/TensorFlowNET.Console/Program.cs

@@ -47,7 +47,7 @@ namespace Tensorflow
             // explaination of constant
             mm.Execute(10, 100 * batchSize, basic.Constant2x3);
 
-            mm.Execute(10, 100 * batchSize, basic.ConstantString);
+            mm.Execute(10, batchSize, basic.ConstantString);
 
             // 100K float variable.
             mm.Execute(10, batchSize, basic.Variable);

src/TensorFlowNET.Console/Tensorflow.Console.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <RootNamespace>Tensorflow</RootNamespace>
     <AssemblyName>Tensorflow</AssemblyName>
     <Platforms>AnyCPU;x64</Platforms>
@@ -11,10 +11,11 @@
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
     <DefineConstants>TRACE;DEBUG</DefineConstants>
+    <PlatformTarget>x64</PlatformTarget>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.3.1" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.4.1" />
   </ItemGroup>
 
   <ItemGroup>

src/TensorFlowNET.Core/APIs/tf.linalg.cs

@@ -13,6 +13,7 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 ******************************************************************************/
+using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
@@ -37,8 +38,8 @@ namespace Tensorflow
             public Tensor matmul(Tensor a, Tensor b)
                 => math_ops.matmul(a, b);
 
-            public Tensor batch_matmul(Tensor x, Tensor y)
-                => gen_math_ops.batch_mat_mul(x, y);
+            public Tensor batch_matmul(Tensor x, Tensor y, bool adj_x = false, bool adj_y = false, string name = null)
+                => math_ops.batch_matmul(x, y, adj_x: adj_x, adj_y: adj_y, name: name);
         }
 
         public Tensor diag(Tensor diagonal, string name = null)
@@ -47,7 +48,32 @@ namespace Tensorflow
         public Tensor matmul(Tensor a, Tensor b)
             => math_ops.matmul(a, b);
 
-        public Tensor batch_matmul(Tensor x, Tensor y)
-            => gen_math_ops.batch_mat_mul(x, y);
+        /// <summary>
+        /// Multiply slices of the two matrices "x" and "y".
+        /// </summary>
+        /// <remarks>
+        /// The `BatchMatMul` operation is embedded into the
+        /// `MatMul` operation on the DLL side. However the expected
+        /// attributes are not the same, hence we need to expose this
+        /// method to have the right args list on the `_apply_op_helper`
+        /// function.
+        ///
+        /// For each rank > 2 the first rank - 2 dimensions are considered
+        /// as fixed, and have to be consistent across the two matrices. A
+        /// common matrix multiplication is then applied over the residual
+        /// 2 dimensions.
+        ///
+        /// e.g.
+        ///     x is (3, 6, 12); y is (3, 12, 6)
+        ///     batch_matmul(x, y) ==> (3, 6, 6)
+        /// </remarks>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="adj_x"></param>
+        /// <param name="adj_y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public Tensor batch_matmul(Tensor x, Tensor y, bool adj_x = false, bool adj_y = false, string name = null)
+            => math_ops.batch_matmul(x, y, adj_x: adj_x, adj_y: adj_y, name: name);
     }
 }

src/TensorFlowNET.Core/APIs/tf.math.cs

@@ -32,6 +32,28 @@ namespace Tensorflow
             /// <returns></returns>
             public Tensor erf(Tensor x, string name = null)
                 => math_ops.erf(x, name);
+
+            /// <summary>
+            /// 
+            /// </summary>
+            /// <param name="arr"></param>
+            /// <param name="weights"></param>
+            /// <param name="minlength"></param>
+            /// <param name="maxlength"></param>
+            /// <param name="dtype"></param>
+            /// <param name="name"></param>
+            /// <param name="axis"></param>
+            /// <param name="binary_output"></param>
+            /// <returns></returns>
+            public Tensor bincount(Tensor arr, Tensor weights = null,
+                Tensor minlength = null,
+                Tensor maxlength = null,
+                TF_DataType dtype = TF_DataType.TF_INT32,
+                string name = null,
+                TensorShape axis = null,
+                bool binary_output = false)
+                => math_ops.bincount(arr, weights: weights, minlength: minlength, maxlength: maxlength,
+                    dtype: dtype, name: name, axis: axis, binary_output: binary_output);
         }
 
         public Tensor abs(Tensor x, string name = null)

src/TensorFlowNET.Core/APIs/tf.sparse.cs

@@ -14,17 +14,18 @@
    limitations under the License.
 ******************************************************************************/
 
+using System;
 using Tensorflow.Framework;
 
 namespace Tensorflow
 {
     public partial class tensorflow
     {
-        public SparseTensor<T> SparseTensor<T>(long[,] indices, T[] values, long[] dense_shape)
-            => new SparseTensor<T>(indices, values, dense_shape);
+        public SparseTensor SparseTensor(long[,] indices, Array values, long[] dense_shape)
+            => new SparseTensor(indices, values, dense_shape);
 
-        public Tensor sparse_tensor_to_dense<T>(SparseTensor<T> sp_input,
-            T default_value = default,
+        public Tensor sparse_tensor_to_dense(SparseTensor sp_input,
+            Array default_value = default,
             bool validate_indices = true,
             string name = null)
             => gen_sparse_ops.sparse_to_dense(sp_input.indices,

src/TensorFlowNET.Core/APIs/tf.strings.cs

@@ -14,6 +14,8 @@
    limitations under the License.
 ******************************************************************************/
 
+using Tensorflow.Framework;
+
 namespace Tensorflow
 {
     public partial class tensorflow
@@ -24,6 +26,30 @@ namespace Tensorflow
         {
             string_ops ops = new string_ops();
 
+            /// <summary>
+            /// Converts all uppercase characters into their respective lowercase replacements.
+            /// </summary>
+            /// <param name="input"></param>
+            /// <param name="encoding"></param>
+            /// <param name="name"></param>
+            /// <returns></returns>
+            public Tensor lower(Tensor input, string encoding = "", string name = null)
+                => ops.lower(input: input, encoding: encoding, name: name);
+
+            /// <summary>
+            /// 
+            /// </summary>
+            /// <param name="input"></param>
+            /// <param name="pattern"></param>
+            /// <param name="rewrite"></param>
+            /// <param name="replace_global"></param>
+            /// <param name="name"></param>
+            /// <returns></returns>
+            public Tensor regex_replace(Tensor input, string pattern, string rewrite,
+                bool replace_global = true, string name = null)
+                => ops.regex_replace(input, pattern, rewrite,
+                    replace_global: replace_global, name: name);
+
             /// <summary>
             /// Return substrings from `Tensor` of strings.
             /// </summary>
@@ -40,6 +66,27 @@ namespace Tensorflow
             public Tensor substr(string input, int pos, int len,
                     string name = null, string @uint = "BYTE")
                 => ops.substr(input, pos, len, @uint: @uint, name: name);
+
+            /// <summary>
+            /// String lengths of `input`.
+            /// </summary>
+            /// <param name="input"></param>
+            /// <param name="name"></param>
+            /// <param name="unit"></param>
+            /// <returns></returns>
+            public Tensor string_length(Tensor input, string name = null, string unit = "BYTE")
+                => ops.string_length(input, name: name, unit: unit);
+
+            public RaggedTensor split(Tensor input, string sep = "", int maxsplit = -1, string name = null)
+                => ops.string_split_v2(input, sep: sep, maxsplit : maxsplit, name : name);
+
+            public (RaggedTensor, RaggedTensor) unicode_decode_with_offsets(Tensor input, string input_encoding, 
+                string errors = "replace", int replacement_char = 0xFFFD, 
+                bool replace_control_characters = false, string name = null)
+                => ops.unicode_decode_with_offsets(input, input_encoding, errors,
+                    replacement_char: replacement_char,
+                    replace_control_characters: replace_control_characters,
+                    name: name);
         }
     }
 }

src/TensorFlowNET.Core/Contexts/Context.AutoMode.cs

@@ -1,90 +0,0 @@
-/*****************************************************************************
-   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-******************************************************************************/
-
-using System;
-using System.Diagnostics;
-using System.Linq;
-using Tensorflow.Eager;
-using static Tensorflow.Binding;
-using Google.Protobuf;
-
-namespace Tensorflow.Contexts
-{
-    /// <summary>
-    /// Environment in which eager operations execute.
-    /// </summary>
-    public sealed partial class Context
-    {
-        // [DebuggerStepThrough]
-        public T RunInAutoMode<T>(Func<T> graphAction, Func<T> eagerAction, params object[] args)
-        {
-            if (tf.Context.has_graph_arg(args))
-            {
-                if (executing_eagerly())
-                {
-                    graph_mode();
-                    var result = graphAction();
-                    restore_mode();
-                    return result;
-                }
-                else
-                {
-                    return graphAction();
-                }
-            }
-            else
-            {
-                if (tf.Context.executing_eagerly())
-                {
-                    return eagerAction();
-                }
-                else
-                {
-                    return graphAction();
-                }
-            }
-        }
-
-        // [DebuggerStepThrough]
-        public Tensors RunInAutoMode2(Func<Tensors> graphAction,
-            Func<Tensors> eagerAction,
-            Action<Operation> recordGradient,
-            Tensors tensors)
-        {
-            if (tf.Context.has_graph_arg(tensors))
-            {
-                if (executing_eagerly())
-                {
-                    graph_mode();
-                    var result = graphAction();
-                    restore_mode();
-                    return result;
-                }
-                else
-                {
-                    var result = graphAction();
-                    if (tf.Runner.MustRecordGradient())
-                        recordGradient(result[0].op);
-                    return result;
-                }
-            }
-            else
-            {
-                return eagerAction();
-            }
-        }
-    }
-}

src/TensorFlowNET.Core/Contexts/Context.ExecuteOp.cs

@@ -0,0 +1,105 @@
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+using Tensorflow.Eager;
+using static Tensorflow.Binding;
+using Google.Protobuf;
+using System.Collections.Generic;
+
+namespace Tensorflow.Contexts
+{
+    /// <summary>
+    /// Environment in which eager operations execute.
+    /// </summary>
+    public sealed partial class Context
+    {
+        // [DebuggerStepThrough]
+        public Tensors ExecuteOp(string OpType, string Name, ExecuteOpArgs args)
+        {
+            Func<Tensors> graphAction = () =>
+            {
+                var keywords = new Dictionary<string, object>();
+                if(args.OpInputArgs != null)
+                {
+                    foreach (var (i, input) in enumerate(args.OpInputArgs))
+                        keywords[$"input_{i}"] = input;
+                }
+
+                if(args.OpAttrs != null)
+                {
+                    foreach (var attr in args.OpAttrs)
+                        keywords[attr.Key] = attr.Value;
+                }
+
+                return tf.OpDefLib._apply_op_helper(OpType, Name, keywords).outputs;
+            };
+
+            Func<Tensors> eagerAction = () =>
+            {
+                return tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(OpType, Name, args.OpInputArgs)
+                {
+                    attrs = args.OpAttrs
+                });
+            };
+
+            if (tf.Context.has_graph_arg(args.OpInputArgs))
+            {
+                if (executing_eagerly())
+                {
+                    graph_mode();
+                    var result = graphAction();
+                    restore_mode();
+                    return result;
+                }
+                else
+                {
+                    var result = graphAction();
+                    if (tf.Runner.MustRecordGradient())
+                    {
+                        var op = result[0].op;
+                        Dictionary<string, object> attrs;
+                        if (args.GetGradientAttrs == null)
+                        {
+                            attrs = new Dictionary<string, object>();
+                            attrs["T"] = op.get_attr<TF_DataType>("T");
+                        }
+                        else
+                        {
+                            attrs = ConvertToDict(args.GetGradientAttrs(op));
+                        }
+                        var args1 = new object[attrs.Count() * 2];
+                        int i = 0;
+                        foreach (var arg in attrs)
+                        {
+                            args1[i] = arg.Key;
+                            args1[i + 1] = arg.Value;
+                            i += 2;
+                        }
+                        tf.Runner.RecordGradient(OpType, op.inputs, args1, op.outputs);
+                    }
+                    return result;
+                }
+            }
+            else
+            {
+                return eagerAction();
+            }
+        }
+    }
+}

src/TensorFlowNET.Core/Contexts/Context.cs

@@ -136,7 +136,10 @@ namespace Tensorflow.Contexts
         public bool has_graph_arg(params object[] args)
         {
             var flatten_args = nest.flatten<object>(args);
-            bool has_graph_arg = false;
+            /*if (flatten_args.Count(x => x.GetType().IsValueType) == flatten_args.Count())
+                return tf.Context.executing_eagerly() == false*/
+
+            bool has_graph_arg = !tf.Context.executing_eagerly();
             foreach (var el in flatten_args)
             {
                 if (el is Tensor tensor && !tensor.IsEagerTensor)

src/TensorFlowNET.Core/Contexts/ExecuteOpArgs.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+
+namespace Tensorflow
+{
+    public class ExecuteOpArgs
+    {
+        public Func<Operation, object> GetGradientAttrs { get; set; }
+        public object[] OpInputArgs { get; set; }
+        public Dictionary<string, object> OpAttrs { get; set; }
+        
+        public ExecuteOpArgs(params object[] inputArgs)
+        {
+            OpInputArgs = inputArgs;
+        }
+
+        public ExecuteOpArgs SetAttributes(object attrs)
+        {
+            OpAttrs = ConvertToDict(attrs);
+            return this;
+        }
+    }
+}

src/TensorFlowNET.Core/Data/DatasetV2.cs

@@ -14,6 +14,7 @@ namespace Tensorflow
     public class DatasetV2 : IDatasetV2
     {
         protected dataset_ops ops = new dataset_ops();
+        public string[] class_names { get; set; }
         public Tensor variant_tensor { get; set; }
 
         public TensorSpec[] structure { get; set; }
@@ -54,7 +55,7 @@ namespace Tensorflow
         public IDatasetV2 optimize(string[] optimizations, string[] optimization_configs)
             => new OptimizeDataset(this, optimizations, optimization_configs: optimization_configs);
 
-        public IDatasetV2 map(Func<Tensor, Tensor> map_func,
+        public IDatasetV2 map(Func<Tensors, Tensors> map_func,
             bool use_inter_op_parallelism = true,
             bool preserve_cardinality = true,
             bool use_legacy_function = false)
@@ -64,9 +65,20 @@ namespace Tensorflow
                 preserve_cardinality: preserve_cardinality,
                 use_legacy_function: use_legacy_function);
 
-        public IDatasetV2 map(Func<Tensors, Tensors> map_func, int num_parallel_calls = -1)
+        public IDatasetV2 map(Func<Tensors, Tensors> map_func, int num_parallel_calls)
             => new ParallelMapDataset(this, map_func, num_parallel_calls: num_parallel_calls);
 
+        public OwnedIterator make_one_shot_iterator()
+        {
+            if (tf.Context.executing_eagerly())
+            {
+                // with ops.colocate_with(self._variant_tensor)
+                return new OwnedIterator(this);
+            }
+
+            throw new NotImplementedException("");
+        }
+
         public IDatasetV2 flat_map(Func<Tensor, IDatasetV2> map_func)
             => new FlatMapDataset(this, map_func);
 
@@ -104,18 +116,7 @@ namespace Tensorflow
         }
 
         public Tensor dataset_cardinality(string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DatasetCardinality", name,
-                    null,
-                    variant_tensor);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("DatasetCardinality", name, new ExecuteOpArgs(variant_tensor));
 
         public override string ToString()
             => $"{GetType().Name} shapes: {string.Join(", ", structure.Select(x => x.shape))}, types: {string.Join(", ", structure.Select(x => "tf." + x.dtype.as_numpy_name()))}";

src/TensorFlowNET.Core/Data/IDatasetV2.cs

@@ -6,6 +6,8 @@ namespace Tensorflow
 {
     public interface IDatasetV2 : IEnumerable<(Tensor, Tensor)>
     {
+        string[] class_names { get; set; }
+
         Tensor variant_tensor { get; set; }
 
         TensorShape[] output_shapes { get; }
@@ -62,13 +64,15 @@ namespace Tensorflow
 
         IDatasetV2 optimize(string[] optimizations, string[] optimization_configs);
 
-        IDatasetV2 map(Func<Tensor, Tensor> map_func,
+        IDatasetV2 map(Func<Tensors, Tensors> map_func,
             bool use_inter_op_parallelism = true,
             bool preserve_cardinality = true,
             bool use_legacy_function = false);
 
         IDatasetV2 map(Func<Tensors, Tensors> map_func,
-            int num_parallel_calls = -1);
+            int num_parallel_calls);
+
+        OwnedIterator make_one_shot_iterator();
 
         IDatasetV2 flat_map(Func<Tensor, IDatasetV2> map_func);
 

src/TensorFlowNET.Core/Data/MapDataset.cs

@@ -10,16 +10,18 @@ namespace Tensorflow
     public class MapDataset : UnaryDataset
     {
         public MapDataset(IDatasetV2 input_dataset,
-            Func<Tensor, Tensor> map_func,
+            Func<Tensors, Tensors> map_func,
             bool use_inter_op_parallelism = true,
             bool preserve_cardinality = false,
             bool use_legacy_function = false) : base(input_dataset)
         {
             var func = new ConcreteFunction($"{map_func.Method.Name}_{Guid.NewGuid()}");
             func.Enter();
-            var input = tf.placeholder(input_dataset.element_spec[0].dtype);
-            var output = map_func(input);
-            func.ToGraph(input, output);
+            var inputs = new Tensors();
+            foreach (var input in input_dataset.element_spec)
+                inputs.Add(tf.placeholder(input.dtype, shape: input.shape));
+            var outputs = map_func(inputs);
+            func.ToGraph(inputs, outputs);
             func.Exit();
 
             structure = func.OutputStructure;

src/TensorFlowNET.Core/Data/OwnedIterator.cs

@@ -26,6 +26,7 @@ namespace Tensorflow
             dataset = dataset.apply_options();
             _dataset = dataset;
             _element_spec = dataset.element_spec;
+            // _flat_output_types = 
             (_iterator_resource, _deleter) = ops.anonymous_iterator_v2(_dataset.output_types, _dataset.output_shapes);
             ops.make_iterator(dataset.variant_tensor, _iterator_resource);
         }

src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs

@@ -15,69 +15,54 @@ namespace Tensorflow.Eager
     /// </summary>
     public partial class EagerRunner
     {
-        int kFastPathExecuteInputStartIndex = 0;
         UnorderedMap<Context, SafeOpHandle> thread_local_eager_operation_map = new UnorderedMap<Context, SafeOpHandle>();
 
-        public Tensor[] TFE_FastPathExecute(Context ctx,
-            string device_name,
-            string opName,
-            string name,
-            Action callbacks,
-            params object[] args)
+        public Tensor[] TFE_FastPathExecute(FastPathOpExecInfo op_exec_info)
         {
-            if (ctx == null)
-                throw new ValueError("This function does not handle the case of the path where " +
-                    "all inputs are not already EagerTensors.");
+            if (op_exec_info.ctx == null)
+                op_exec_info.ctx = tf.Context;
+            if (string.IsNullOrEmpty(op_exec_info.device_name))
+                op_exec_info.device_name = tf.Context.DeviceName;
 
-            int args_size = args.Length;
             var attr_list_sizes = new Dictionary<string, long>();
 
-            FastPathOpExecInfo op_exec_info = new FastPathOpExecInfo()
-            {
-                ctx = ctx,
-                args = args,
-                device_name = device_name,
-                op_name = opName,
-                name = name,
-            };
-
             op_exec_info.run_gradient_callback = HasAccumulatorOrTape();
-            op_exec_info.run_post_exec_callbacks = callbacks != null;
+            op_exec_info.run_post_exec_callbacks = op_exec_info.callbacks != null;
             op_exec_info.run_callbacks = op_exec_info.run_gradient_callback || op_exec_info.run_post_exec_callbacks;
 
             var status = tf.Status;
-            using var op = GetOp(ctx, opName, status);
+            using var op = GetOp(op_exec_info.ctx, op_exec_info.op_name, status);
 
-            var op_def = tf.get_default_graph().GetOpDef(opName);
+            var op_def = tf.get_default_graph().GetOpDef(op_exec_info.op_name);
 
             var flattened_attrs = new List<object>(op_def.Attr.Count * 2);
             var flattened_inputs = new List<Tensor>(op_def.InputArg.Count);
 
             // Set non-inferred attrs, including setting defaults if the attr is passed in
             // as None.
-            for (int i = kFastPathExecuteInputStartIndex + op_def.InputArg.Count; i < args_size; i += 2)
+            if(op_exec_info.attrs != null)
             {
-                var attr_name = args[i].ToString();
-                var attr_value = args[i + 1];
-
-                var attr = op_def.Attr.FirstOrDefault(x => x.Name == attr_name);
-                if (attr != null)
+                foreach (var attr1 in op_exec_info.attrs)
                 {
-                    flattened_attrs.Add(attr_name);
-                    flattened_attrs.Add(attr_value);
+                    var attr = op_def.Attr.FirstOrDefault(x => x.Name == attr1.Key);
+                    if (attr != null)
+                    {
+                        flattened_attrs.Add(attr.Name);
+                        flattened_attrs.Add(attr1.Value);
 
-                    SetOpAttrWithDefaults(ctx, op, attr, attr_name, attr_value, attr_list_sizes, status);
-                    status.Check(true);
+                        SetOpAttrWithDefaults(op_exec_info.ctx, op, attr, attr.Name, attr1.Value, attr_list_sizes, status);
+                        status.Check(true);
+                    }
                 }
             }
 
-            c_api.TFE_OpSetDevice(op, device_name, status.Handle);
+            c_api.TFE_OpSetDevice(op, op_exec_info.device_name, status.Handle);
             status.Check(true);
 
             // Add inferred attrs and inputs.
             for (int i = 0; i < op_def.InputArg.Count; i++)
             {
-                var input = args[kFastPathExecuteInputStartIndex + i];
+                var input = op_exec_info.args[i];
                 var input_arg = op_def.InputArg[i];
                 if (!string.IsNullOrEmpty(input_arg.NumberAttr))
                 {
@@ -92,7 +77,7 @@ namespace Tensorflow.Eager
 
                     if (len > 0)
                     {
-                        var fast_input_array = (object[])args[i];
+                        var fast_input_array = (object[])op_exec_info.args[i];
                         // First item adds the type attr.
                         if (!AddInputToOp(fast_input_array[i], true, input_arg, flattened_attrs, flattened_inputs, op, status))
                             return null;
@@ -136,7 +121,7 @@ namespace Tensorflow.Eager
                 else
                 {
                     // The item is a single item.
-                    AddInputToOp(args[i], true, input_arg, flattened_attrs, flattened_inputs, op, status);
+                    AddInputToOp(op_exec_info.args[i], true, input_arg, flattened_attrs, flattened_inputs, op, status);
                 }
             }
 
@@ -164,7 +149,7 @@ namespace Tensorflow.Eager
             if (op_exec_info.run_callbacks)
             {
                 RunCallbacks(op_exec_info,
-                    kFastPathExecuteInputStartIndex + op_def.InputArg.Count(),
+                    op_def.InputArg.Count(),
                     flattened_inputs.ToArray(), flattened_attrs.ToArray(), flat_result);
             }
 

src/TensorFlowNET.Core/Eager/FastPathOpExecInfo.cs

@@ -1,6 +1,8 @@
-using Tensorflow.Contexts;
+using System;
+using System.Collections.Generic;
+using Tensorflow.Contexts;
 
-namespace Tensorflow.Eager
+namespace Tensorflow
 {
     public class FastPathOpExecInfo
     {
@@ -9,8 +11,17 @@ namespace Tensorflow.Eager
         public string op_name { get; set; }
         public string name { get; set; }
         public object[] args { get; set; }
+        public Dictionary<string, object> attrs { get; set; }
         public bool run_gradient_callback { get; set; }
         public bool run_post_exec_callbacks { get; set; }
         public bool run_callbacks { get; set; }
+        public Action callbacks { get; set; }
+
+        public FastPathOpExecInfo(string opName, string name, params object[] inputArgs)
+        {
+            this.op_name = opName;
+            this.name = name;
+            this.args = inputArgs;
+        }
     }
 }

src/TensorFlowNET.Core/Eager/IEagerRunner.cs

@@ -16,12 +16,7 @@ namespace Tensorflow.Eager
             TF_DataType default_dtype = TF_DataType.DtInvalid,
             object[] args = null);
 
-        Tensor[] TFE_FastPathExecute(Context ctx,
-            string device_name,
-            string opName,
-            string name,
-            Action callbacks,
-            params object[] args);
+        Tensor[] TFE_FastPathExecute(FastPathOpExecInfo op_exec_info);
 
         Tensor[] TFE_Execute(Context ctx,
             string device_name,

src/TensorFlowNET.Core/Framework/Models/TensorSpec.cs

@@ -15,7 +15,7 @@ namespace Tensorflow.Framework.Models
             if (_shape.ndim == 0)
                 throw new ValueError("Unbatching a tensor is only supported for rank >= 1");
 
-            return new TensorSpec(_shape.dims[1..], _dtype);
+            return new TensorSpec(_shape.dims.Skip(1).ToArray(), _dtype);
         }
 
         public TensorSpec _batch(int dim = -1)

src/TensorFlowNET.Core/Framework/sparse_tensor.py.cs

@@ -1,63 +0,0 @@
-using System;
-using System.Linq;
-using static Tensorflow.Binding;
-
-namespace Tensorflow.Framework
-{
-    /// <summary>
-    /// Represents a sparse tensor.
-    /// </summary>
-    public class SparseTensor<T> : CompositeTensor, _TensorLike
-    {
-        long[,] _indices;
-        public Tensor indices;
-
-        T[] _values;
-        public Tensor values;
-
-        long[] _dense_shape;
-        public Tensor dense_shape;
-
-        TensorShape _shape;
-        public TensorShape shape => _shape;
-
-        public TF_DataType dtype => dtypes.as_dtype(typeof(T));
-
-        public SparseTensor(long[,] indices_, T[] values_, long[] dense_shape_)
-        {
-            tf_with(ops.name_scope(null, "SparseTensor", new { }), delegate
-            {
-                indices = ops.convert_to_tensor(
-                    indices_, name: "indices", dtype: dtypes.int64);
-                values = ops.convert_to_tensor(values_, name: "values");
-                dense_shape = ops.convert_to_tensor(
-                    dense_shape_, name: "dense_shape", dtype: dtypes.int64);
-            });
-
-            _indices = indices_;
-            _values = values_;
-            _dense_shape = dense_shape_;
-
-            var indices_shape = indices.TensorShape.with_rank(2);
-            var values_shape = values.TensorShape.with_rank(1);
-            var dense_shape_shape = dense_shape.TensorShape.with_rank(1);
-
-            indices_shape["0"].merge_with(values_shape[0]);
-            indices_shape["1"].merge_with(dense_shape_shape[0]);
-
-            _shape = new TensorShape(_dense_shape.Select(x => Convert.ToInt32(x)).ToArray());
-        }
-    }
-
-    public interface _TensorLike
-    {
-    }
-
-    public static class sparse_tensor_extension
-    {
-        public static bool is_sparse(this _TensorLike x)
-        {
-            return x.GetType().Name.Contains("SparseTensor");
-        }
-    }
-}

src/TensorFlowNET.Core/Framework/tensor_shape.cs

@@ -44,14 +44,14 @@ namespace Tensorflow.Framework
                 return true;
             }
 
-            if (other.is_sparse())
+            if (other.IsSparseTensor)
             {
                 return self.dtype.is_compatible_with(other.dtype);
             }
 
             return self.dtype.is_compatible_with(other.dtype) &&
                    _shape_is_compatible_0dim(self.shape, other.shape) &&
-                   !self.is_sparse();
+                   !self.IsSparseTensor;
         }
 
         public static Dimension dimension_at_index(TensorShape shape, int index)

src/TensorFlowNET.Core/Gradients/image_grad.cs

@@ -30,7 +30,7 @@ namespace Tensorflow.Gradients
             var shape = new TensorShape(image.shape.Skip(1).Take(2).ToArray());
             Tensor image_shape = null;
             if (shape.is_fully_defined())
-                image_shape = constant_op.constant(image.shape[1..3]);
+                image_shape = constant_op.constant(image.shape.Skip(1).Take(2).ToArray());
             else
                 image_shape = array_ops.shape(image)["1:3"];
 

src/TensorFlowNET.Core/Gradients/math_grad.cs

@@ -291,23 +291,23 @@ namespace Tensorflow.Gradients
             var b = math_ops.conj(op.inputs[1]);
             if (!t_a && !t_b)
             {
-                grad_a = gen_math_ops.batch_mat_mul(grad, b, adj_y: true);
-                grad_b = gen_math_ops.batch_mat_mul(a, grad, adj_x: true);
+                grad_a = math_ops.batch_matmul(grad, b, adj_y: true);
+                grad_b = math_ops.batch_matmul(a, grad, adj_x: true);
             }
             else if (!t_a && t_b)
             {
-                grad_a = gen_math_ops.batch_mat_mul(grad, b);
-                grad_b = gen_math_ops.batch_mat_mul(grad, a, adj_x: true);
+                grad_a = math_ops.batch_matmul(grad, b);
+                grad_b = math_ops.batch_matmul(grad, a, adj_x: true);
             }
             else if (t_a && !t_b)
             {
-                grad_a = gen_math_ops.batch_mat_mul(grad, b);
-                grad_b = gen_math_ops.batch_mat_mul(grad, a, adj_x: true);
+                grad_a = math_ops.batch_matmul(grad, b);
+                grad_b = math_ops.batch_matmul(grad, a, adj_x: true);
             }
             else if (t_a && t_b)
             {
-                grad_a = gen_math_ops.batch_mat_mul(b, grad, adj_x: true, adj_y: true);
-                grad_b = gen_math_ops.batch_mat_mul(grad, a, adj_x: true, adj_y: true);
+                grad_a = math_ops.batch_matmul(b, grad, adj_x: true, adj_y: true);
+                grad_b = math_ops.batch_matmul(grad, a, adj_x: true, adj_y: true);
             }
 
             return new Tensor[] { grad_a, grad_b };

src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling1DArgs.cs

@@ -0,0 +1,34 @@
+namespace Tensorflow.Keras.ArgsDefinition
+{
+    public class Pooling1DArgs : LayerArgs
+    {
+        /// <summary>
+        /// The pooling function to apply, e.g. `tf.nn.max_pool2d`.
+        /// </summary>
+        public IPoolFunction PoolFunction { get; set; }
+
+        /// <summary>
+        /// specifying the size of the pooling window.
+        /// </summary>
+        public int PoolSize { get; set; }
+
+        /// <summary>
+        /// specifying the strides of the pooling operation.
+        /// </summary>
+        public int Strides { 
+            get { return _strides.HasValue ? _strides.Value : PoolSize; }
+            set { _strides = value; } 
+        }
+        private int? _strides = null;
+
+        /// <summary>
+        /// The padding method, either 'valid' or 'same'.
+        /// </summary>
+        public string Padding { get; set; } = "valid";
+
+        /// <summary>
+        /// one of `channels_last` (default) or `channels_first`.
+        /// </summary>
+        public string DataFormat { get; set; }
+    }
+}

src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/PreprocessingLayerArgs.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.ArgsDefinition
+{
+    public class PreprocessingLayerArgs : LayerArgs
+    {
+    }
+}

src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.ArgsDefinition
+{
+    public class TextVectorizationArgs : PreprocessingLayerArgs
+    {
+        public Func<Tensor, Tensor> Standardize { get; set; }
+        public string Split { get; set; } = "standardize";
+        public int MaxTokens { get; set; } = -1;
+        public string OutputMode { get; set; } = "int";
+        public int OutputSequenceLength { get; set; } = -1;
+        public string[] Vocabulary { get; set; }
+    }
+}

src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs

@@ -40,37 +40,16 @@ namespace Tensorflow.Operations
         /// <param name="parameters"></param>
         /// <returns></returns>
         public static Tensor conv2d(Conv2dParams parameters)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Conv2D", parameters.Name,
-                    null,
-                    parameters.Input, parameters.Filter,
-                    "strides", parameters.Strides,
-                    "use_cudnn_on_gpu", parameters.UseCudnnOnGpu,
-                    "padding", parameters.Padding,
-                    "explicit_paddings", parameters.ExplicitPaddings,
-                    "data_format", parameters.DataFormat,
-                    "dilations", parameters.Dilations);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Conv2D", name: parameters.Name, args: new
-            {
-                input = parameters.Input,
-                filter = parameters.Filter,
-                strides = parameters.Strides,
-                padding = parameters.Padding,
-                use_cudnn_on_gpu = parameters.UseCudnnOnGpu,
-                explicit_paddings = parameters.ExplicitPaddings,
-                data_format = parameters.DataFormat,
-                dilations = parameters.Dilations
-            });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Conv2D", parameters.Name, new ExecuteOpArgs(parameters.Input, parameters.Filter)
+                .SetAttributes(new
+                {
+                    strides = parameters.Strides,
+                    padding = parameters.Padding,
+                    use_cudnn_on_gpu = parameters.UseCudnnOnGpu,
+                    explicit_paddings = parameters.ExplicitPaddings,
+                    data_format = parameters.DataFormat,
+                    dilations = parameters.Dilations
+                }));
 
         /// <summary>
         /// Computes the gradients of convolution with respect to the filter.
@@ -83,43 +62,16 @@ namespace Tensorflow.Operations
             string data_format = "NHWC",
             int[] dilations = null,
             string name = null)
-        {
-            if (explicit_paddings == null)
-                explicit_paddings = new int[0];
-            if (dilations == null)
-                dilations = new int[] { 1, 1, 1, 1 };
-
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Conv2DBackpropFilter", name,
-                    null,
-                    input, filter_sizes, out_backprop,
-                    "strides", strides,
-                    "use_cudnn_on_gpu", use_cudnn_on_gpu,
-                    "padding", padding,
-                    "explicit_paddings", explicit_paddings,
-                    "data_format", data_format,
-                    "dilations", dilations);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Conv2DBackpropFilter", name: name, args: new
-            {
-                input,
-                filter_sizes,
-                out_backprop,
-                strides,
-                padding,
-                use_cudnn_on_gpu,
-                explicit_paddings,
-                data_format,
-                dilations
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("Conv2DBackpropFilter", name, new ExecuteOpArgs(input, filter_sizes, out_backprop)
+                    .SetAttributes(new
+                    {
+                        strides,
+                        padding,
+                        use_cudnn_on_gpu,
+                        explicit_paddings = explicit_paddings ?? new int[0],
+                        data_format,
+                        dilations = dilations ?? new int[] { 1, 1, 1, 1 }
+                    }));
 
         /// <summary>
         /// Computes the gradients of convolution with respect to the input.
@@ -132,99 +84,29 @@ namespace Tensorflow.Operations
             string data_format = "NHWC",
             int[] dilations = null,
             string name = null)
-        {
-            if (explicit_paddings == null)
-                explicit_paddings = new int[0];
-            if (dilations == null)
-                dilations = new int[] { 1, 1, 1, 1 };
-
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Conv2DBackpropInput", name,
-                    null,
-                    input_sizes, filter, out_backprop,
-                    "strides", strides,
-                    "use_cudnn_on_gpu", use_cudnn_on_gpu,
-                    "padding", padding,
-                    "explicit_paddings", explicit_paddings,
-                    "data_format", data_format,
-                    "dilations", dilations);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Conv2DBackpropInput", name: name, args: new
-            {
-                input_sizes,
-                filter,
-                out_backprop,
-                strides,
-                padding,
-                use_cudnn_on_gpu,
-                explicit_paddings,
-                data_format,
-                dilations
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("Conv2DBackpropInput", name, new ExecuteOpArgs(input_sizes, filter, out_backprop)
+                    .SetAttributes(new
+                    {
+                        strides,
+                        padding,
+                        use_cudnn_on_gpu,
+                        explicit_paddings = explicit_paddings ?? new int[0],
+                        data_format,
+                        dilations = dilations ?? new int[] { 1, 1, 1, 1 }
+                    }));
 
         public static Tensor bias_add(Tensor value,
             IVariableV1 bias,
             string data_format = null,
             string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "BiasAdd", name,
-                    null,
-                    value, bias,
-                    "data_format", data_format);
-
-                return results[0];
-            }
-
-            if (data_format == null)
-                data_format = "NHWC";
-
-            var _op = tf.OpDefLib._apply_op_helper("BiasAdd", name: name, args: new
-            {
-                value,
-                bias,
-                data_format
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("BiasAdd", name, new ExecuteOpArgs(value, bias)
+                    .SetAttributes(new { data_format = data_format ?? "NHWC" }));
 
         public static Tensor bias_add_grad(Tensor out_backprop,
             string data_format = "NHWC",
             string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "BiasAddGrad", name,
-                    null,
-                    out_backprop,
-                    "data_format", data_format);
-
-                return results[0];
-            }
-
-            if (data_format == null)
-                data_format = "NHWC";
-
-            var _op = tf.OpDefLib._apply_op_helper("BiasAddGrad", name: name, args: new
-            {
-                out_backprop,
-                data_format
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("BiasAddGrad", name, new ExecuteOpArgs(out_backprop)
+                    .SetAttributes(new { data_format = data_format ?? "NHWC" }));
 
         /// <summary>
         /// Computes exponential linear: <c>exp(features) - 1</c> if &amp;lt; 0, <c>features</c> otherwise.
@@ -269,29 +151,19 @@ namespace Tensorflow.Operations
         }
 
         public static Tensor[] fused_batch_norm_grad_v3(FusedBatchNormParams @params)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("FusedBatchNormGradV3", name: @params.Name,
-                    args: new 
-                    {
-                        y_backprop = @params.YBackprop,
-                        x = @params.X,
-                        scale = @params.Scale,
-                        reserve_space_1 = @params.ReserveSpace1,
-                        reserve_space_2 = @params.ReserveSpace2,
-                        reserve_space_3 = @params.ReserveSpace3,
-                        epsilon = @params.Epsilon,
-                        data_format = @params.DataFormat,
-                        is_training = @params.IsTraining
-                    }).outputs, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "FusedBatchNormGradV3", @params.Name,
-                    null,
-                    @params.YBackprop, @params.X, @params.Scale,
-                    @params.ReserveSpace1, @params.ReserveSpace2, @params.ReserveSpace3,
-                    "epsilon", @params.Epsilon, 
-                    "data_format", @params.DataFormat, 
-                    "is_training", @params.IsTraining),
-                @params.YBackprop);
+            => tf.Context.ExecuteOp("FusedBatchNormGradV3", @params.Name,
+                new ExecuteOpArgs(@params.YBackprop,
+                    @params.X,
+                    @params.Scale,
+                    @params.ReserveSpace1,
+                    @params.ReserveSpace2,
+                    @params.ReserveSpace3)
+                .SetAttributes(new
+                {
+                    epsilon = @params.Epsilon,
+                    data_format = @params.DataFormat,
+                    is_training = @params.IsTraining
+                }));
 
         public static Tensor[] fused_batch_norm(Tensor x,
                 Tensor scale,
@@ -328,39 +200,8 @@ namespace Tensorflow.Operations
             string data_format = "NHWC",
             bool is_training = true,
             string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "FusedBatchNormV3", name,
-                    null,
-                    x,
-                    scale,
-                    offset,
-                    mean,
-                    variance,
-                    "epsilon", epsilon,
-                    "exponential_avg_factor", exponential_avg_factor,
-                    "data_format", data_format,
-                    "is_training", is_training);
-
-                return results;
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("FusedBatchNormV3", name: name, args: new
-            {
-                x,
-                scale,
-                offset,
-                mean,
-                variance,
-                epsilon,
-                data_format,
-                is_training
-            });
-
-            return _op.outputs;
-        }
+                => tf.Context.ExecuteOp("FusedBatchNormV3", name, new ExecuteOpArgs(x, scale, offset, mean, variance)
+                    .SetAttributes(new { epsilon, data_format, is_training }));
 
         /// <summary>
         /// Local Response Normalization.
@@ -388,14 +229,7 @@ namespace Tensorflow.Operations
         }
 
         public static Tensor log_softmax(Tensor logits, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("LogSoftmax", name: name,
-                    args: new { logits }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "LogSoftmax", name,
-                    null,
-                    logits).FirstOrDefault(),
-                logits);
+            => tf.Context.ExecuteOp("LogSoftmax", name, new ExecuteOpArgs(logits));
 
         /// <summary>
         /// Says whether the targets are in the top `K` predictions.
@@ -418,19 +252,8 @@ namespace Tensorflow.Operations
         }
 
         public static Tensor leaky_relu(Tensor features, float alpha = 0.2f, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("LeakyRelu", name: name,
-                    args: new
-                    {
-                        features,
-                        alpha
-                    }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "LeakyRelu", name,
-                    null,
-                    features,
-                    "alpha", alpha).FirstOrDefault(),
-                features);
+            => tf.Context.ExecuteOp("LeakyRelu", name,
+                new ExecuteOpArgs(features).SetAttributes(new { alpha }));
 
         public static Tensor max_pool(Tensor input,
             int[] ksize,
@@ -438,63 +261,25 @@ namespace Tensorflow.Operations
             string padding,
             string data_format = "NHWC",
             string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MaxPool", name,
-                    null,
-                    input,
-                    "ksize", ksize,
-                    "strides", strides,
-                    "padding", padding,
-                    "data_format", data_format);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("MaxPool", name: name, args: new
-            {
-                input,
-                ksize,
-                strides,
-                padding,
-                data_format,
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("MaxPool", name, new ExecuteOpArgs(input)
+                    .SetAttributes(new
+                    {
+                        ksize,
+                        strides,
+                        padding,
+                        data_format
+                    }));
 
         public static Tensor max_pool_grad(Tensor orig_input, Tensor orig_output, Tensor grad, int[] ksize, int[] strides, string padding,
             string data_format = "NHWC", string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MaxPoolGrad", name,
-                    null,
-                    orig_input, orig_output, grad,
-                    "ksize", ksize,
-                    "strides", strides,
-                    "padding", padding,
-                    "data_format", data_format);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("MaxPoolGrad", name: name, args: new
-            {
-                orig_input,
-                orig_output,
-                grad,
-                ksize,
-                strides,
-                padding,
-                data_format
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("MaxPoolGrad", name, new ExecuteOpArgs(orig_input, orig_output, grad)
+                    .SetAttributes(new
+                    {
+                        ksize,
+                        strides,
+                        padding,
+                        data_format
+                    }));
 
         public static Tensor[] top_kv2(Tensor input, int k, bool sorted = true, string name = null)
         {
@@ -509,68 +294,14 @@ namespace Tensorflow.Operations
         }
 
         public static Tensor relu_grad(Tensor gradients, Tensor features, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ReluGrad", name,
-                    null,
-                    gradients, features);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ReluGrad", name: name, args: new
-            {
-                gradients,
-                features
-            });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("ReluGrad", name, new ExecuteOpArgs(gradients, features));
 
         public static Tensor leaky_relu_grad(Tensor gradients, Tensor features, float alpha = 0.2f, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "LeakyReluGrad", name,
-                    null,
-                    gradients, features,
-                    "alpha", alpha);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("LeakyReluGrad", name: name, args: new
-            {
-                gradients,
-                features,
-                alpha
-            });
-
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("LeakyReluGrad", name, new ExecuteOpArgs(gradients, features)
+                .SetAttributes(new { alpha }));
 
         public static Tensor softmax(Tensor logits, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Softmax", name,
-                    null,
-                    logits);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Softmax", name: name, args: new
-            {
-                logits
-            });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(logits));
 
         /// <summary>
         /// Computes softmax cross entropy cost and gradients to backpropagate.
@@ -581,23 +312,9 @@ namespace Tensorflow.Operations
         /// <returns></returns>
         public static (Tensor, Tensor) softmax_cross_entropy_with_logits(Tensor features, Tensor labels, string name = null)
         {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "SoftmaxCrossEntropyWithLogits", name,
-                    null,
-                    features, labels);
-
-                return (results[0], results[1]);
-            }
+            var results = tf.Context.ExecuteOp("SoftmaxCrossEntropyWithLogits", name, new ExecuteOpArgs(features, labels));
 
-            var _op = tf.OpDefLib._apply_op_helper("SoftmaxCrossEntropyWithLogits", name: name, args: new
-            {
-                features,
-                labels
-            });
-
-            return (_op.outputs[0], _op.outputs[1]);
+            return (results[0], results[1]);
         }
 
         /// <summary>
@@ -629,21 +346,9 @@ namespace Tensorflow.Operations
         /// </remarks>
         public static (Tensor loss, Tensor backprop) sparse_softmax_cross_entropy_with_logits(Tensor features, Tensor labels, string name = "SparseSoftmaxCrossEntropyWithLogits")
         {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "SparseSoftmaxCrossEntropyWithLogits", name,
-                    null,
-                    features, labels);
-
-                return (results[0], results[1]);
-            }
-
-            var op = tf.OpDefLib._apply_op_helper("SparseSoftmaxCrossEntropyWithLogits", name: name, args: new { features, labels });
-            int _idx = 0;
-            var loss = op.outputs[_idx++];
-            var backprop = op.outputs[_idx++];
-            return (loss, backprop);
+            var results = tf.Context.ExecuteOp("SparseSoftmaxCrossEntropyWithLogits", name, new ExecuteOpArgs(features, labels));
+
+            return (results[0], results[1]);
         }
 
         /// <summary>
@@ -653,35 +358,9 @@ namespace Tensorflow.Operations
         /// <param name="name">A name for the operation (optional).</param>
         /// <returns>A `Tensor`. Has the same type as `features`.</returns>
         public static Tensor relu(Tensor features, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Relu", name,
-                    null,
-                    features);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Relu", name: name, args: new { features });
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features));
 
         public static Tensor tanh(Tensor x, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Tanh", name,
-                    null,
-                    x);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Tanh", name: name, args: new { x });
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(x));
     }
 }

src/TensorFlowNET.Core/Operations/OpDefLibrary.cs

@@ -68,10 +68,10 @@ namespace Tensorflow
                 string _scope_name = scope;
 
                 // Perform input type inference
-                foreach (var input_arg in op_def.InputArg)
+                foreach (var (i, input_arg) in enumerate(op_def.InputArg))
                 {
                     var input_name = input_arg.Name;
-
+                    
                     if (keywords.ContainsKey(input_name))
                         values = keywords[input_name];
                     else if (keywords.ContainsKey(input_name + "_"))
@@ -79,6 +79,10 @@ namespace Tensorflow
                         input_name += "_";
                         values = keywords[input_name];
                     }
+                    else if (keywords.ContainsKey($"input_{i}"))
+                    {
+                        values = keywords[$"input_{i}"];
+                    }
                     else
                         throw new TypeError("No argument for input " + input_name);
 

src/TensorFlowNET.Core/Operations/array_ops.cs

@@ -57,20 +57,8 @@ namespace Tensorflow
         ///    gradients in some corner cases.
         /// </remarks>
         public static Tensor prevent_gradient(Tensor input, string message = "", string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "PreventGradient", name,
-                    null,
-                    input,
-                    "message", message);
-                return results[0];
-            }
-
-            var op = tf.OpDefLib._apply_op_helper("PreventGradient", name: name, args: new { input, message });
-            return op.output;
-        }
+            => tf.Context.ExecuteOp("PreventGradient", name, new ExecuteOpArgs(input)
+                .SetAttributes(new { message }));
 
         internal static Tensor constant(object value,
             TF_DataType dtype = TF_DataType.DtInvalid,
@@ -737,44 +725,27 @@ namespace Tensorflow
         public static Tensor strided_slice_grad(Tensor shape, Tensor begin, Tensor end, Tensor strides, Tensor dy,
             long begin_mask = 0, long end_mask = 0, long ellipsis_mask = 0, long new_axis_mask = 0,
             long shrink_axis_mask = 0, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("StridedSliceGrad", name, new
+            => tf.Context.ExecuteOp("StridedSliceGrad", name,
+                new ExecuteOpArgs(shape, begin, end, strides, dy)
+                {
+                    GetGradientAttrs = (op) => new
+                    {
+                        T = op.get_attr<TF_DataType>("T"),
+                        Index = op.get_attr<TF_DataType>("Index"),
+                        begin_mask = op.get_attr<long>("begin_mask"),
+                        end_mask = op.get_attr<long>("end_mask"),
+                        ellipsis_mask = op.get_attr<long>("ellipsis_mask"),
+                        new_axis_mask = op.get_attr<long>("new_axis_mask"),
+                        shrink_axis_mask = op.get_attr<long>("shrink_axis_mask")
+                    }
+                }.SetAttributes(new
                 {
-                    shape,
-                    begin,
-                    end,
-                    strides,
-                    dy,
                     begin_mask,
                     end_mask,
                     ellipsis_mask,
                     new_axis_mask,
                     shrink_axis_mask
-                }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "StridedSliceGrad", name,
-                    null,
-                    shape, begin, end, strides, dy,
-                    "begin_mask", begin_mask,
-                    "end_mask", end_mask,
-                    "ellipsis_mask", ellipsis_mask,
-                    "new_axis_mask", new_axis_mask,
-                    "shrink_axis_mask", shrink_axis_mask).FirstOrDefault(),
-                (op) =>
-                {
-                    var attrs = new object[]
-                    {
-                        "T", op.get_attr<TF_DataType>("T"),
-                        "Index", op.get_attr<TF_DataType>("Index"),
-                        "begin_mask", op.get_attr<long>("begin_mask"),
-                        "end_mask", op.get_attr<long>("end_mask"),
-                        "ellipsis_mask", op.get_attr<long>("ellipsis_mask"),
-                        "new_axis_mask", op.get_attr<long>("new_axis_mask"),
-                        "shrink_axis_mask", op.get_attr<long>("shrink_axis_mask")
-                    };
-                    tf.Runner.RecordGradient("StridedSliceGrad", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(shape, begin, end, strides, dy));
+                }));
 
         /// <summary>
         /// Removes dimensions of size 1 from the shape of a tensor.
@@ -809,38 +780,17 @@ namespace Tensorflow
                 int num_cols = -1,
                 float padding_value = 0,
                 string align = "RIGHT_LEFT")
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MatrixDiagV3", name,
-                    null,
-                    diagonal, k, num_rows, num_cols, padding_value,
-                    "align", align);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("MatrixDiagV3", name, 
+                new ExecuteOpArgs(diagonal, k, num_rows, num_cols, padding_value)
+                    .SetAttributes(new { align }));
 
         public static Tensor matrix_set_diag(Tensor input,
             Tensor diagonal,
             string name = "set_diag",
             int k = 0,
             string align = "RIGHT_LEFT")
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MatrixSetDiagV3", name,
-                    null,
-                    input, diagonal, k,
-                    "align", align);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("MatrixSetDiagV3", name, new ExecuteOpArgs(input, diagonal, k)
+                    .SetAttributes(new { align }));
 
         /// <summary>
         /// Computes the shape of a broadcast given symbolic shapes.
@@ -969,27 +919,14 @@ namespace Tensorflow
             => gen_array_ops.slice(input, begin, size, name: name);
 
         public static Tensor slice(Tensor input, Tensor begin, Tensor size, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("Slice", name, new
-                {
-                    input,
-                    begin,
-                    size
-                }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Slice", name,
-                    null,
-                    input, begin, size).FirstOrDefault(),
-                (op) =>
+            => tf.Context.ExecuteOp("Slice", name, new ExecuteOpArgs(input, begin, size)
+            {
+                GetGradientAttrs = (op) => new
                 {
-                    var attrs = new object[]
-                    {
-                        "T", op.get_attr<TF_DataType>("T"),
-                        "Index", op.get_attr<int>("Index")
-                    };
-                    tf.Runner.RecordGradient("Slice", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(input, begin, size));
+                    T = op.get_attr<TF_DataType>("T"),
+                    Index = op.get_attr<int>("Index")
+                }
+            });
 
         public static Tensor stack(object values, int axis = 0, string name = "stack")
         {

src/TensorFlowNET.Core/Operations/bitwise_ops.cs

@@ -94,20 +94,7 @@ namespace Tensorflow.Operations
         /// <param name="name"></param>
         /// <returns></returns>
         Tensor unary_op(Tensor x, string opName, string name)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    opName, name,
-                    null,
-                    x);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper(opName, name, args: new { x });
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp(opName, name, new ExecuteOpArgs(x));
 
         /// <summary>
         /// Helper method to invoke binary operator with specified name.
@@ -118,21 +105,7 @@ namespace Tensorflow.Operations
         /// <param name="name"></param>
         /// <returns></returns>
         Tensor binary_op(Tensor x, Tensor y, string opName, string name)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    opName, name,
-                    null,
-                    x, y);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper(opName, name, args: new { x, y });
-            return _op.output;
-        }
-
+            => tf.Context.ExecuteOp(opName, name, new ExecuteOpArgs(x, y));
         #endregion
     }
 }

src/TensorFlowNET.Core/Operations/dataset_ops.cs

@@ -8,26 +8,10 @@ namespace Tensorflow
     public class dataset_ops
     {
         public Tensor tensor_dataset(Tensor[] components, TensorShape[] output_shapes, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
+            => tf.Context.ExecuteOp("TensorDataset", name, new ExecuteOpArgs()
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "TensorDataset", name,
-                    null,
-                    new object[]
-                    {
-                        components,
-                        "output_shapes", output_shapes
-                    });
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("TensorDataset",
-                name: name,
-                args: new { components, output_shapes });
-
-            return _op.output;
-        }
+                OpInputArgs = new object[] { components }
+            }.SetAttributes(new { output_shapes }));
 
         /// <summary>
         /// Creates a dataset that emits each dim-0 slice of `components` once.
@@ -37,192 +21,62 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public Tensor tensor_slice_dataset(Tensor[] components, TensorShape[] output_shapes, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
+            => tf.Context.ExecuteOp("TensorSliceDataset", name, new ExecuteOpArgs()
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "TensorSliceDataset", name,
-                    null,
-                    new object[]
-                    {
-                        components,
-                        "output_shapes", output_shapes
-                    });
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("TensorSliceDataset",
-                name: name,
-                args: new { components, output_shapes });
-
-            return _op.outputs[0];
-        }
+                OpInputArgs = new object[] { components }
+            }.SetAttributes(new { output_shapes }));
 
         public Tensor range_dataset(Tensor start, Tensor stop, Tensor step, TF_DataType[] output_types, TensorShape[] output_shapes, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "RangeDataset", name,
-                    null,
-                    start, stop, step,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("RangeDataset", name, new ExecuteOpArgs(start, stop, step)
+                .SetAttributes(new { output_types, output_shapes }));
 
         public Tensor repeat_dataset(Tensor input_dataset, Tensor count, TF_DataType[] output_types, TensorShape[] output_shapes, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "RepeatDataset", name,
-                    null,
-                    input_dataset, count,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("RepeatDataset", name, new ExecuteOpArgs(input_dataset, count)
+                .SetAttributes(new { output_types, output_shapes }));
 
         public Tensor shard_dataset(Tensor input_dataset, Tensor num_shards, Tensor index,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             bool require_non_empty = false, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ShardDataset", name,
-                    null,
-                    input_dataset, num_shards, index,
-                    "require_non_empty", require_non_empty,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ShardDataset", name, new ExecuteOpArgs(input_dataset, num_shards, index)
+                    .SetAttributes(new { require_non_empty, output_types, output_shapes }));
 
         public Tensor zip_dataset(Tensor[] input_datasets,
             TF_DataType[] output_types,
             TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ZipDataset", name,
-                    null,
-                    new object[]
-                    {
-                        input_datasets,
-                        "output_types", output_types,
-                        "output_shapes", output_shapes
-                    });
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ZipDataset", name, new ExecuteOpArgs()
+                {
+                    OpInputArgs = new object[] { input_datasets }
+                }.SetAttributes(new { output_types, output_shapes }));
 
         public Tensor shuffle_dataset_v3(Tensor input_dataset, Tensor buffer_size,
             Tensor seed, Tensor seed2, Tensor seed_generator,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             bool reshuffle_each_iteration = true,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ShuffleDatasetV3", name,
-                    null,
-                    input_dataset, buffer_size,
-                    seed, seed2, seed_generator,
-                    "reshuffle_each_iteration", reshuffle_each_iteration,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ShuffleDatasetV3", name, new ExecuteOpArgs(input_dataset, buffer_size, seed, seed2, seed_generator)
+                    .SetAttributes(new { reshuffle_each_iteration, output_types, output_shapes }));
 
         public Tensor skip_dataset(Tensor input_dataset, Tensor count,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "SkipDataset", name,
-                    null,
-                    input_dataset, count,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("SkipDataset", name, new ExecuteOpArgs(input_dataset, count)
+                    .SetAttributes(new { output_types, output_shapes }));
 
         public Tensor dummy_seed_generator(string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DummySeedGenerator", name,
-                    null);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("DummySeedGenerator", name, new ExecuteOpArgs());
 
         public Tensor concatenate_dataset(Tensor input_dataset, Tensor another_dataset,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ConcatenateDataset", name,
-                    null,
-                    input_dataset, another_dataset,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ConcatenateDataset",
-                name: name,
-                args: new { input_dataset, another_dataset, output_types, output_shapes });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("ConcatenateDataset", name, new ExecuteOpArgs(input_dataset, another_dataset)
+                    .SetAttributes(new { output_types, output_shapes }));
 
         public Tensor cache_dataset_v2(Tensor input_dataset, Tensor filename, Tensor cache,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "CacheDatasetV2", name,
-                    null,
-                    input_dataset, filename, cache,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("CacheDatasetV2", name, new ExecuteOpArgs(input_dataset, filename, cache)
+                    .SetAttributes(new { output_types, output_shapes }));
 
         /// <summary>
         /// Creates a dataset that batches `batch_size` elements from `input_dataset`.
@@ -240,21 +94,9 @@ namespace Tensorflow
             TF_DataType[] output_types, TensorShape[] output_shapes,
             bool parallel_copy = false,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "BatchDatasetV2", name,
-                    null,
-                    input_dataset, buffer_size, drop_remainder,
-                    "parallel_copy", parallel_copy,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("BatchDatasetV2", name, 
+                    new ExecuteOpArgs(input_dataset, buffer_size, drop_remainder)
+                        .SetAttributes(new { parallel_copy, output_types, output_shapes }));
 
         /// <summary>
         /// 
@@ -262,17 +104,7 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public Tensor dummy_memory_cache(string name = "")
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DummyMemoryCache", name,
-                    null);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("DummyMemoryCache", name, new ExecuteOpArgs());
 
         /// <summary>
         /// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
@@ -290,22 +122,14 @@ namespace Tensorflow
             int? slack_period = 0,
             bool legacy_autotune = true,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "PrefetchDataset", name,
-                    null,
-                    input_dataset, buffer_size,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes,
-                    "slack_period", slack_period,
-                    "legacy_autotune", legacy_autotune);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("PrefetchDataset", name, new ExecuteOpArgs(input_dataset, buffer_size)
+                    .SetAttributes(new
+                    {
+                        output_types,
+                        output_shapes,
+                        slack_period,
+                        legacy_autotune
+                    }));
 
         /// <summary>
         /// Creates a dataset that contains `count` elements from the `input_dataset`.
@@ -319,20 +143,8 @@ namespace Tensorflow
         public Tensor take_dataset(Tensor input_dataset, Tensor count,
             TF_DataType[] output_types, TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "TakeDataset", name,
-                    null,
-                    input_dataset, count,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("TakeDataset", name, new ExecuteOpArgs(input_dataset, count)
+                    .SetAttributes(new { output_types, output_shapes }));
 
         /// <summary>
         /// Creates a dataset by applying optimizations to `input_dataset`.
@@ -348,24 +160,13 @@ namespace Tensorflow
             TF_DataType[] output_types, TensorShape[] output_shapes,
             string[] optimization_configs = null,
             string name = null)
-        {
-            if (optimization_configs == null)
-                optimization_configs = new string[0];
-
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "OptimizeDataset", name,
-                    null,
-                    input_dataset, optimizations,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes,
-                    "optimization_configs", optimization_configs);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("OptimizeDataset", name, new ExecuteOpArgs(input_dataset, optimizations)
+                    .SetAttributes(new
+                    {
+                        output_types,
+                        output_shapes,
+                        optimization_configs = optimization_configs ?? new string[0]
+                    }));
 
         /// <summary>
         /// Identity transformation that models performance.
@@ -381,22 +182,14 @@ namespace Tensorflow
             TF_DataType[] output_types, TensorShape[] output_shapes,
             AutotuneAlgorithm algorithm, long cpu_budget,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ModelDataset", name,
-                    null,
-                    input_dataset,
-                    "algorithm", algorithm,
-                    "cpu_budget", cpu_budget,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ModelDataset", name, new ExecuteOpArgs(input_dataset)
+                    .SetAttributes(new
+                    {
+                        algorithm,
+                        cpu_budget,
+                        output_types,
+                        output_shapes
+                    }));
 
         /// <summary>
         /// A container for an iterator resource.
@@ -407,17 +200,9 @@ namespace Tensorflow
         /// <returns>A tuple of `Tensor` objects (handle, deleter).</returns>
         public (Tensor, Tensor) anonymous_iterator_v2(TF_DataType[] output_types, TensorShape[] output_shapes, string name = null)
         {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "AnonymousIteratorV2", name,
-                    null,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return (results[0], results[1]);
-            }
-
-            throw new NotImplementedException("");
+            var results = tf.Context.ExecuteOp("AnonymousIteratorV2", name, 
+                new ExecuteOpArgs().SetAttributes(new { output_types, output_shapes }));
+            return (results[0], results[1]);
         }
 
         /// <summary>
@@ -427,19 +212,8 @@ namespace Tensorflow
         /// <param name="iterator"></param>
         /// <param name="name"></param>
         /// <returns>The created Operation.</returns>
-        public ITensorOrOperation make_iterator(Tensor dataset, Tensor iterator, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MakeIterator", name,
-                    null,
-                    dataset, iterator);
-                return null;
-            }
-
-            throw new NotImplementedException("");
-        }
+        public void make_iterator(Tensor dataset, Tensor iterator, string name = null)
+            => tf.Context.ExecuteOp("MakeIterator", name, new ExecuteOpArgs(dataset, iterator));
 
         /// <summary>
         /// 
@@ -450,23 +224,15 @@ namespace Tensorflow
         /// <returns></returns>
         public Tensor map_dataset(Tensor dataset, ConcreteFunction f, TF_DataType[] output_types, TensorShape[] output_shapes,
             bool use_inter_op_parallelism = true, bool preserve_cardinality = false, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "MapDataset", name,
-                    null,
-                    dataset, new Tensor[0],
-                    "f", f,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes,
-                    "use_inter_op_parallelism", use_inter_op_parallelism,
-                    "preserve_cardinality", preserve_cardinality);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("MapDataset", name, new ExecuteOpArgs(dataset, new Tensor[0])
+                    .SetAttributes(new
+                    {
+                        f,
+                        output_types,
+                        output_shapes,
+                        use_inter_op_parallelism,
+                        preserve_cardinality
+                    }));
 
         /// <summary>
         /// Creates a dataset that applies `f` to the outputs of `input_dataset`.
@@ -479,21 +245,8 @@ namespace Tensorflow
         /// <returns></returns>
         public Tensor flat_map_dataset(Tensor dataset, ConcreteFunction f, TF_DataType[] output_types, TensorShape[] output_shapes,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "FlatMapDataset", name,
-                    null,
-                    dataset, new Tensor[0],
-                    "f", f,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("FlatMapDataset", name, new ExecuteOpArgs(dataset, new Tensor[0])
+                    .SetAttributes(new { f, output_types, output_shapes }));
 
         /// <summary>
         /// Creates a dataset that applies `f` to the outputs of `input_dataset`.
@@ -512,24 +265,17 @@ namespace Tensorflow
             string deterministic = "default",
             bool preserve_cardinality = false,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ParallelMapDatasetV2", name,
-                    null,
-                    dataset, new Tensor[0], num_parallel_calls,
-                    "f", f,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes,
-                    "use_inter_op_parallelism", use_inter_op_parallelism,
-                    "deterministic", deterministic,
-                    "preserve_cardinality", preserve_cardinality);
-                return results[0];
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ParallelMapDatasetV2", name, 
+                    new ExecuteOpArgs(dataset, new Tensor[0], num_parallel_calls)
+                        .SetAttributes(new
+                        {
+                            f,
+                            output_types,
+                            output_shapes,
+                            use_inter_op_parallelism,
+                            deterministic,
+                            preserve_cardinality
+                        }));
 
         /// <summary>
         /// A container for an iterator resource.
@@ -538,19 +284,8 @@ namespace Tensorflow
         /// <param name="deleter"></param>
         /// <param name="name"></param>
         /// <returns>The created Operation.</returns>
-        public ITensorOrOperation delete_iterator(Tensor handle, Tensor deleter, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DeleteIterator", name,
-                    null,
-                    handle, deleter);
-                return null;
-            }
-
-            throw new NotImplementedException("");
-        }
+        public void delete_iterator(Tensor handle, Tensor deleter, string name = null)
+            => tf.Context.ExecuteOp("DeleteIterator", name, new ExecuteOpArgs(handle, deleter));
 
         /// <summary>
         /// Gets the next output from the given iterator .
@@ -561,19 +296,7 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public Tensor[] iterator_get_next(Tensor iterator, TF_DataType[] output_types, TensorShape[] output_shapes, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "IteratorGetNext", name,
-                    null,
-                    iterator,
-                    "output_types", output_types,
-                    "output_shapes", output_shapes);
-                return results;
-            }
-
-            throw new NotImplementedException("");
-        }
+            => tf.Context.ExecuteOp("IteratorGetNext", name, new ExecuteOpArgs(iterator)
+                .SetAttributes(new { output_types, output_shapes }));
     }
 }

src/TensorFlowNET.Core/Operations/gen_array_ops.cs

@@ -45,20 +45,7 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor concat_v2<T, Ta>(T[] values, Ta axis, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ConcatV2", name,
-                    null,
-                    values, axis);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ConcatV2", name: name, args: new { values, axis });
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis));
 
         public static Tensor concat_v2(Tensor[] values, Tensor axis, string name = null)
         {
@@ -72,14 +59,7 @@ namespace Tensorflow
         }
 
         public static Tensor concat_v2(Tensor[] values, int axis, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("ConcatV2", name: name,
-                    args: new { values, axis }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ConcatV2", name,
-                    null,
-                    values, axis).FirstOrDefault(),
-                values);
+            => tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis));
 
         private static Tensor concat_v2_eager_fallback<T1, T2>(T1[] values, T2 axis, string name, Context ctx)
         {
@@ -131,38 +111,11 @@ namespace Tensorflow
         ///   </code>
         /// </remarks>
         public static Tensor diag(Tensor diagonal, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Diag", name,
-                    null,
-                    diagonal);
-
-                return results[0];
-            }
-
-            var op = tf.OpDefLib._apply_op_helper("Diag", name: name, args: new { diagonal });
-
-            return op.output;
-        }
+            => tf.Context.ExecuteOp("Diag", name, new ExecuteOpArgs(diagonal));
 
         public static Tensor expand_dims(Tensor input, int axis, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ExpandDims", name,
-                    null,
-                    input, tf.convert_to_tensor(axis));
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ExpandDims", name: name, args: new { input, dim = axis });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("ExpandDims", name, new ExecuteOpArgs(input, axis)
+                .SetAttributes(new { dim = axis }));
 
         public static Tensor gather_v2<T1, T2>(T1 @params, T2 indices, int axis, string name = null)
         {
@@ -202,14 +155,10 @@ namespace Tensorflow
         }
 
         public static Tensor pack(Tensor[] values, int axis = 0, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("Pack", name, new { values, axis }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Pack", name,
-                    null,
-                    values,
-                    "axis", axis).FirstOrDefault(),
-                values, axis);
+            => tf.Context.ExecuteOp("Pack", name, new ExecuteOpArgs()
+            {
+                OpInputArgs = new object[] { values }
+            }.SetAttributes(new { axis }));
 
         /// <summary>
         /// Return a tensor with the same shape and contents as the input tensor or value.
@@ -217,29 +166,7 @@ namespace Tensorflow
         /// <param name="input"></param>
         /// <param name="name"></param>
         public static Tensor identity(Tensor input, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Identity", name,
-                    null,
-                    input);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Identity", name, new { input });
-            
-            if (tf.Runner.MustRecordGradient())
-            {
-                tf.Runner.RecordGradient("Identity", _op.inputs, new object[] 
-                {
-                    "T", _op.get_attr<TF_DataType>("T")
-                }, _op.outputs);
-            }                
-
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("Identity", name, new ExecuteOpArgs(input));
 
         public static Tensor invert_permutation(Tensor x, string name = null)
         {
@@ -256,21 +183,7 @@ namespace Tensorflow
         }
 
         public static Tensor rank(Tensor input, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Rank", name,
-                    null,
-                    input);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Rank", name: name, args: new { input });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Rank", name, new ExecuteOpArgs(input));
 
         /// <summary>
         /// Creates a tensor filled with a scalar value.
@@ -280,20 +193,7 @@ namespace Tensorflow
         /// <param name="name">A name for the operation (optional).</param>
         /// <returns>A `Tensor`. Has the same type as `value`.</returns>
         public static Tensor fill<T>(Tensor dims, T value, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Fill", name,
-                    null,
-                    dims, value);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Fill", name, new { dims, value });
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("Fill", name, new ExecuteOpArgs(dims, value));
 
         /// <summary>
         /// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
@@ -304,19 +204,8 @@ namespace Tensorflow
         /// <returns>A tuple of `Tensor` objects (r0, r1).</returns>
         public static (Tensor, Tensor) broadcast_gradient_args(Tensor s0, Tensor s1, string name = "")
         {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "BroadcastGradientArgs", name,
-                    null,
-                    s0, s1);
-
-                return (results[0], results[1]);
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("BroadcastGradientArgs", name, new { s0, s1 });
-
-            return (_op.outputs[0], _op.outputs[1]);
+            var results = tf.Context.ExecuteOp("BroadcastGradientArgs", name, new ExecuteOpArgs(s0, s1));
+            return (results[0], results[1]);
         }
 
         public static Tensor reverse<T>(Tensor tensor, T axis, string name = null)
@@ -326,31 +215,10 @@ namespace Tensorflow
         }
 
         public static Tensor reshape<T>(Tensor tensor, T shape, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("Reshape", name, new { tensor, shape }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Reshape", name,
-                    null,
-                    tensor, shape).FirstOrDefault(),
-                tensor, shape);
+            => tf.Context.ExecuteOp("Reshape", name, new ExecuteOpArgs(tensor, shape));
 
         public static Tensor reshape(Tensor tensor, object[] shape, string name = null)
-        {
-            try
-            {
-                return tf.Context.RunInAutoMode(()
-                     => tf.OpDefLib._apply_op_helper("Reshape", name, new { tensor, shape }).output, ()
-                     => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                         "Reshape", name,
-                         null,
-                         tensor, shape).FirstOrDefault(),
-                     tensor, shape);
-            }
-            catch (InvalidArgumentError ex)
-            {
-                return reshape_eager_fallback(tensor, shape, name, tf.Context);
-            }
-        }
+            => tf.Context.ExecuteOp("Reshape", name, new ExecuteOpArgs(tensor, shape));
 
         private static Tensor reshape_eager_fallback(Tensor tensor, object[] shape, string name, Context ctx)
         {
@@ -400,21 +268,8 @@ namespace Tensorflow
             TF_DataType dtype = TF_DataType.DtInvalid,
             int axis = -1,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "OneHot", name,
-                    null,
-                    indices, depth, on_value, off_value,
-                    "axis", axis);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("OneHot", name, new { indices, depth, on_value, off_value, axis });
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("OneHot", name, new ExecuteOpArgs(indices, depth, on_value, off_value)
+                    .SetAttributes(new { axis }));
 
         /// <summary>
         /// A placeholder op that passes through `input` when its output is not fed.
@@ -430,35 +285,10 @@ namespace Tensorflow
         }
 
         public static Tensor select<Tx, Ty>(Tensor condition, Tx x, Ty y, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Select", name,
-                    null,
-                    condition, x, y);
-
-                return results[0];
-            }
+            => tf.Context.ExecuteOp("Select", name, new ExecuteOpArgs(condition, x, y));
 
-            var _op = tf.OpDefLib._apply_op_helper("Select", name, new { condition, t = x, e = y });
-            return _op.outputs[0];
-        }
         public static Tensor select_v2<Tx, Ty>(Tensor condition, Tx x, Ty y, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "SelectV2", name,
-                    null,
-                    condition, x, y);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("SelectV2", name, new { condition, t = x, e = y });
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("SelectV2", name, new ExecuteOpArgs(condition, x, y));
 
         public static Tensor scatter_nd(Tensor indices, Tensor updates, Tensor[] shape, string name = null)
         {
@@ -467,15 +297,8 @@ namespace Tensorflow
         }
 
         public static Tensor shape(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("Shape", name,
-                    new { input, out_type }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Shape", name,
-                    null,
-                    input,
-                    "out_type", out_type).FirstOrDefault(),
-                input);
+            => tf.Context.ExecuteOp("Shape", name, new ExecuteOpArgs(input)
+                .SetAttributes(new { out_type }));
 
         /// <summary>
         /// Returns shape of tensors.
@@ -485,21 +308,10 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor[] shape_n(Tensor[] input, TF_DataType out_type = TF_DataType.TF_INT32, string name = null)
-        {
-            if (tf.executing_eagerly())
+            => tf.Context.ExecuteOp("ShapeN", name, new ExecuteOpArgs()
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ShapeN", name,
-                    null,
-                    input,
-                    "out_type", out_type);
-
-                return results;
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ShapeN", name, new { input, out_type });
-            return _op.outputs;
-        }
+                OpInputArgs = new object[] { input }
+            }.SetAttributes(new { out_type }));
 
         public static Tensor size(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string name = null)
         {
@@ -542,72 +354,23 @@ namespace Tensorflow
 
         public static Tensor[] split_v(Tensor value, Tensor size_splits, 
             int axis, int num_split, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "SplitV", name,
-                    null,
-                    value, size_splits, axis,
-                    "num_split", num_split);
-
-                return results;
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("SplitV", name, new { split_dim = axis, value, num_split });
-            return _op.outputs;
-        }
+                => tf.Context.ExecuteOp("SplitV", name, new ExecuteOpArgs(value, size_splits, axis)
+                    .SetAttributes(new { num_split }));
 
         public static Tensor tile(Tensor input, Tensor multiples, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("Tile", name, new { input, multiples }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Tile", name,
-                    null,
-                    input, multiples).FirstOrDefault(),
-                input, multiples);
+            => tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, multiples));
 
         public static Tensor tile(Tensor input, object[] multiples, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("Tile", name, new { input, multiples }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Tile", name,
-                    null,
-                    input, multiples).FirstOrDefault(),
-                input, multiples);
+            => tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, multiples));
 
         public static Tensor transpose<T1>(Tensor x, T1 perm, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Transpose", name,
-                    null,
-                    x, perm);
-
-                return results[0];
-            }
-            var _op = tf.OpDefLib._apply_op_helper("Transpose", name, new { x, perm });
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Transpose", name, new ExecuteOpArgs(x, perm));
 
         public static Tensor ones_like(Tensor x, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("OnesLike", name, new { x }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "OnesLike", name,
-                    null,
-                    x).FirstOrDefault(),
-                x);
+            => tf.Context.ExecuteOp("OnesLike", name, new ExecuteOpArgs(x));
 
         public static Tensor zeros_like(Tensor x, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("ZerosLike", name, new { x }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ZerosLike", name,
-                    null,
-                    x).FirstOrDefault(),
-                x);
+            => tf.Context.ExecuteOp("ZerosLike", name, new ExecuteOpArgs(x));
 
         public static Tensor stop_gradient(Tensor x, string name = null)
         {
@@ -623,53 +386,32 @@ namespace Tensorflow
             long new_axis_mask = 0,
             long shrink_axis_mask = 0,
             string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("StridedSlice", name, new
-                {
-                    input,
-                    begin,
-                    end,
-                    strides,
-                    begin_mask,
-                    end_mask,
-                    ellipsis_mask,
-                    new_axis_mask,
-                    shrink_axis_mask
-                }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "StridedSlice", name,
-                    null,
-                    input, begin, end, strides,
-                    "begin_mask", begin_mask,
-                    "end_mask", end_mask,
-                    "ellipsis_mask", ellipsis_mask,
-                    "new_axis_mask", new_axis_mask,
-                    "shrink_axis_mask", shrink_axis_mask).FirstOrDefault(),
-                input, begin, end, strides);
-
-        public static Operation resource_strided_slice_assign(Tensor input, Tensor begin, Tensor end, Tensor strides, Tensor value,
+                => tf.Context.ExecuteOp("StridedSlice", name, new ExecuteOpArgs(input, begin, end, strides)
+                    .SetAttributes(new
+                    {
+                        begin_mask,
+                        end_mask,
+                        ellipsis_mask,
+                        new_axis_mask,
+                        shrink_axis_mask
+                    }));
+
+        public static Tensor resource_strided_slice_assign(Tensor input, Tensor begin, Tensor end, Tensor strides, Tensor value,
             int begin_mask = 0,
             int end_mask = 0,
             int ellipsis_mask = 0,
             int new_axis_mask = 0,
             int shrink_axis_mask = 0,
             string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("ResourceStridedSliceAssign", name, new
-                {
-                    input, begin, end, strides, value,
-                    begin_mask, end_mask, ellipsis_mask, new_axis_mask, shrink_axis_mask
-                }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResourceStridedSliceAssign", name,
-                    null,
-                    input, begin, end, strides, value,
-                    "begin_mask", begin_mask,
-                    "end_mask", end_mask,
-                    "ellipsis_mask", ellipsis_mask,
-                    "new_axis_mask", new_axis_mask,
-                    "shrink_axis_mask", shrink_axis_mask).FirstOrDefault(),
-                input, begin, end, strides, value);
+                => tf.Context.ExecuteOp("ResourceStridedSliceAssign", name, new ExecuteOpArgs(input, begin, end, strides, value)
+                    .SetAttributes(new
+                    {
+                        begin_mask,
+                        end_mask,
+                        ellipsis_mask,
+                        new_axis_mask,
+                        shrink_axis_mask
+                    }));
 
         public static Tensor strided_slice<T>(Tensor input, T[] begin, T[] end, T[] strides,
             int begin_mask = 0,
@@ -707,23 +449,8 @@ namespace Tensorflow
         /// <param name="name"> A name for the operation (optional).</param>
         /// <returns> A `Tensor`. Has the same type as `input`.</returns>
         public static Tensor squeeze(Tensor input, int[] axis = null, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Squeeze", name,
-                    null,
-                    input,
-                    "squeeze_dims", axis);
-
-                return results[0];
-            }
-
-            if (axis == null) axis = new int[0];
-            var _op = tf.OpDefLib._apply_op_helper("Squeeze", name, args: new { input, squeeze_dims = axis });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("Squeeze", name, new ExecuteOpArgs(input)
+                .SetAttributes(new { squeeze_dims = axis }));
 
         /// <summary>
         /// Return the shape of s0 op s1 with broadcast.
@@ -749,20 +476,6 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor broadcast_to<T>(Tensor input, T shape, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "BroadcastTo", name,
-                    null,
-                    input, shape);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("BroadcastTo", name, args: new { input, shape, name });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("BroadcastTo", name, new ExecuteOpArgs(input, shape));
     }
 }

src/TensorFlowNET.Core/Operations/gen_image_ops.cs

@@ -70,38 +70,17 @@ namespace Tensorflow
             float acceptable_fraction = 1,
             string dct_method = "",
             string name = null)
-        {
-            // Add nodes to the TensorFlow graph.
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DecodeJpeg", name,
-                    null,
-                    contents,
-                    "channels", channels,
-                    "ratio", ratio,
-                    "fancy_upscaling", fancy_upscaling,
-                    "try_recover_truncated", try_recover_truncated,
-                    "acceptable_fraction", acceptable_fraction,
-                    "dct_method", dct_method);
-                return results[0];
-            }
-            else
-            {
-                var _op = tf.OpDefLib._apply_op_helper("DecodeJpeg", name: name, args: new
-                {
-                    contents,
-                    channels,
-                    ratio,
-                    fancy_upscaling,
-                    try_recover_truncated,
-                    acceptable_fraction,
-                    dct_method
-                });
-
-                return _op.outputs[0];
-            }
-        }
+                => tf.Context.ExecuteOp("DecodeJpeg", name,
+                    new ExecuteOpArgs(contents).SetAttributes(
+                    new
+                    {
+                        channels,
+                        ratio,
+                        fancy_upscaling,
+                        try_recover_truncated,
+                        acceptable_fraction,
+                        dct_method
+                    }));
 
         public static Tensor decode_gif(Tensor contents,
             string name = null)
@@ -171,99 +150,36 @@ namespace Tensorflow
             bool align_corners = false,
             bool half_pixel_centers = false,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResizeBilinear", name,
-                    null,
-                    images, size,
-                    "align_corners", align_corners,
-                    "half_pixel_centers", half_pixel_centers);
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ResizeBilinear", name: name, args: new
-            {
-                images,
-                size,
-                align_corners
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("ResizeBilinear", name,
+                    new ExecuteOpArgs(images, size).SetAttributes(new
+                    {
+                        align_corners,
+                        half_pixel_centers
+                    }));
 
         public static Tensor resize_bicubic(Tensor images,
             Tensor size,
             bool align_corners = false,
             bool half_pixel_centers = false,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResizeBicubic", name,
-                    null,
-                    images, size,
-                    "align_corners", align_corners,
-                    "half_pixel_centers", half_pixel_centers);
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ResizeBicubic", name: name, args: new
-            {
-                images,
-                size,
-                align_corners
-            });
-
-            return _op.outputs[0];
-        }
-
+                => tf.Context.ExecuteOp("ResizeBicubic", name, 
+                    new ExecuteOpArgs(images, size).SetAttributes(new { align_corners, half_pixel_centers }));
+        
         public static Tensor resize_nearest_neighbor<Tsize>(Tensor images, Tsize size, bool align_corners = false,
             bool half_pixel_centers = false, string name = null)
-            => tf.Context.RunInAutoMode(()
-                => tf.OpDefLib._apply_op_helper("ResizeNearestNeighbor", name: name, args: new
-                {
-                    images,
-                    size,
-                    align_corners,
-                    half_pixel_centers
-                }).output, ()
-                => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResizeNearestNeighbor", name,
-                    null,
-                    images, size,
-                    "align_corners", align_corners,
-                    "half_pixel_centers", half_pixel_centers).FirstOrDefault(),
-                images);
+                => tf.Context.ExecuteOp("ResizeNearestNeighbor", name, 
+                    new ExecuteOpArgs(images, size).SetAttributes(new { align_corners, half_pixel_centers }));
 
         public static Tensor resize_nearest_neighbor_grad(Tensor grads, Tensor size, bool align_corners = false,
             bool half_pixel_centers = false, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("ResizeNearestNeighborGrad", name, new
+                => tf.Context.ExecuteOp("ResizeNearestNeighborGrad", name, new ExecuteOpArgs(grads, size)
                 {
-                    grads,
-                    size,
-                    align_corners,
-                    half_pixel_centers
-                }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResizeNearestNeighborGrad", name,
-                    null,
-                    grads, size,
-                    "align_corners", align_corners,
-                    "half_pixel_centers", half_pixel_centers).FirstOrDefault(),
-                (op) =>
-                {
-                    var attrs = new object[]
+                    GetGradientAttrs = (op) => new
                     {
-                        "T", op.get_attr<TF_DataType>("T"),
-                        "align_corners", op.get_attr<bool>("align_corners"),
-                        "half_pixel_centers", op.get_attr<bool>("half_pixel_centers")
-                    };
-                    tf.Runner.RecordGradient("ResizeNearestNeighborGrad", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(grads, size));
+                        T = op.get_attr<TF_DataType>("T"),
+                        align_corners = op.get_attr<bool>("align_corners"),
+                        half_pixel_centers = op.get_attr<bool>("half_pixel_centers")
+                    }
+                }.SetAttributes(new { align_corners, half_pixel_centers }));
     }
 }

src/TensorFlowNET.Core/Operations/gen_logging_ops.cs

@@ -25,10 +25,9 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(
                     "Assert", name,
-                    null,
-                    new object[] { condition, data, summarize });
+                    new object[] { condition, data, summarize }));
 
                 return results[0];
             }

src/TensorFlowNET.Core/Operations/gen_math_ops.eager.cs

@@ -6,13 +6,6 @@ namespace Tensorflow
     public static partial class gen_math_ops
     {
         public static Tensor mul(IntPtr x, IntPtr y, string name = null)
-        {
-            var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                "Mul", name,
-                null,
-                x, y);
-
-            return results[0];
-        }
+            => tf.Context.ExecuteOp("Mul", name, new ExecuteOpArgs(x, y));
     }
 }

src/TensorFlowNET.Core/Operations/gen_random_ops.cs

@@ -29,31 +29,8 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor random_standard_normal(Tensor shape, TF_DataType dtype = TF_DataType.DtInvalid, int? seed = null, int? seed2 = null, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "RandomStandardNormal", name,
-                    null,
-                    shape,
-                    "seed", seed,
-                    "seed2", seed2,
-                    "dtype", dtype);
-
-                return results[0];
-            }
-
-            if (!seed.HasValue)
-                seed = 0;
-            if (!seed2.HasValue)
-                seed2 = 0;
-
-            var _op = tf.OpDefLib._apply_op_helper("RandomStandardNormal",
-                name: name,
-                args: new { shape, dtype, seed, seed2 });
-
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("RandomStandardNormal", name, new ExecuteOpArgs(shape)
+                .SetAttributes(new { dtype, seed = seed ?? 0, seed2 = seed2 ?? 0 }));
 
         /// <summary>
         /// Outputs random integers from a uniform distribution.
@@ -89,31 +66,8 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor random_uniform(Tensor shape, TF_DataType dtype, int? seed = 0, int? seed2 = 0, string name = null)
-        {
-            if (!seed.HasValue)
-                seed = 0;
-            if (!seed2.HasValue)
-                seed2 = 0;
-
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "RandomUniform", name,
-                    null,
-                    shape,
-                    "seed", seed,
-                    "seed2", seed2,
-                    "dtype", dtype);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("RandomUniform",
-                name: name,
-                args: new { shape, dtype, seed, seed2 });
-
-            return _op.outputs[0];
-        }
+            => tf.Context.ExecuteOp("RandomUniform", name, new ExecuteOpArgs(shape)
+                .SetAttributes(new { dtype, seed = seed ?? 0, seed2 = seed2 ?? 0 }));
 
         /// <summary>
         /// 
@@ -125,25 +79,7 @@ namespace Tensorflow
         /// <returns></returns>
         public static Tensor random_shuffle(Tensor value, int seed = 0, int seed2 = 0,
             string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "RandomShuffle", name,
-                    null,
-                    value,
-                    "seed", seed,
-                    "seed2", seed2);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("RandomShuffle",
-                name: name,
-                args: new { value, seed, seed2 });
-
-            return _op.output;
-        }
+                => tf.Context.ExecuteOp("RandomShuffle", name, new ExecuteOpArgs(value, seed, seed2));
 
         /// <summary>
         /// Outputs random values from a truncated normal distribution.
@@ -156,31 +92,8 @@ namespace Tensorflow
         /// <returns></returns>
         public static Tensor truncated_normal(Tensor shape, TF_DataType dtype, int? seed = 0,
             int? seed2 = 0, string name = null)
-        {
-            if (!seed.HasValue)
-                seed = 0;
-            if (!seed2.HasValue)
-                seed2 = 0;
-
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "TruncatedNormal", name,
-                    null,
-                    shape,
-                    "seed", seed,
-                    "seed2", seed2,
-                    "dtype", dtype);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("TruncatedNormal",
-                name: name,
-                args: new { shape, dtype, seed, seed2 });
-
-            return _op.output;
-        }
+                => tf.Context.ExecuteOp("TruncatedNormal", name, new ExecuteOpArgs(shape)
+                    .SetAttributes(new { dtype, seed = seed ?? 0, seed2 = seed2 ?? 0 }));
 
         public static Tensor multinomial(Tensor logits, int num_samples, int? seed = 0,
             int? seed2 = 0, TF_DataType output_dtype = TF_DataType.TF_INT64, string name = null)

src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs

@@ -24,10 +24,8 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "AssignSubVariableOp", name,
-                    null,
-                    resource, value);
+                tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(
+                    "AssignSubVariableOp", name, resource, value));
 
                 return null;
             }
@@ -46,10 +44,8 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "AssignAddVariableOp", name,
-                    null,
-                    resource, value);
+                tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("AssignAddVariableOp", name,
+                    resource, value));
 
                 return null;
             }
@@ -63,10 +59,8 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "AssignVariableOp", name,
-                    null,
-                    resource, value);
+                tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("AssignVariableOp", name,
+                    resource, value));
 
                 return null;
             }
@@ -80,10 +74,8 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "VarIsInitializedOp", name,
-                    null,
-                    resource);
+                var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("VarIsInitializedOp", name,
+                    resource));
 
                 return results[0];
             }
@@ -107,14 +99,17 @@ namespace Tensorflow
         {
             if (tf.Context.executing_eagerly())
             {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "VarHandleOp", name,
-                    null,
-                    "container", container,
-                    "shared_name", shared_name,
-                    "dtype", dtype,
-                    "shape", shape.dims,
-                    "allowed_devices", new string[0]);
+                var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("VarHandleOp", name)
+                {
+                    attrs = ConvertToDict(new
+                    {
+                        dtype,
+                        shape = shape.dims,
+                        container,
+                        shared_name,
+                        allowed_devices = new string[0]
+                    })
+                });
 
                 return results[0];
             }
@@ -131,26 +126,8 @@ namespace Tensorflow
         }
 
         public static Tensor destroy_resource_op(Tensor resource, bool ignore_lookup_error = true, string name = null)
-        {
-            if (tf.Context.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "DestroyResourceOp", name,
-                    null,
-                    resource,
-                    "ignore_lookup_error", ignore_lookup_error);
-
-                return results.Length == 0 ? null : results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("DestroyResourceOp", name, new
-            {
-                resource,
-                ignore_lookup_error
-            });
-
-            return _op.output;
-        }
+            => tf.Context.ExecuteOp("DestroyResourceOp", name, 
+                new ExecuteOpArgs(resource).SetAttributes(new { ignore_lookup_error }));
 
         /// <summary>
         /// Reads the value of a variable.
@@ -160,26 +137,8 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor read_variable_op(Tensor resource, TF_DataType dtype, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ReadVariableOp", name,
-                    null,
-                    resource,
-                    "dtype", dtype);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ReadVariableOp", name, new
-            {
-                resource,
-                dtype
-            });
-
-            return _op.output;
-        }
+        => tf.Context.ExecuteOp("ReadVariableOp", name, new ExecuteOpArgs(resource)
+            .SetAttributes(new { dtype }));
 
         public static Tensor resource_gather(Tensor resource, Tensor indices, TF_DataType dtype,
             int batch_dims = 0, bool validate_indices = true, string name = null)

src/TensorFlowNET.Core/Operations/math_ops.cs

@@ -45,21 +45,7 @@ namespace Tensorflow
             => gen_math_ops.add(x, y, name);
 
         public static Tensor add_v2(Tensor x, Tensor y, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("AddV2", name, new { x, y }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "AddV2", name,
-                    null,
-                    x, y).FirstOrDefault(),
-                (op) =>
-                {
-                    var attrs = new object[]
-                    {
-                        "T", op.get_attr<TF_DataType>("T")
-                    };
-                    tf.Runner.RecordGradient("AddV2", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(x, y));
+            => tf.Context.ExecuteOp("AddV2", name, new ExecuteOpArgs(x, y));
 
         public static Tensor add_v2<Tx, Ty>(Tx x, Ty y, string name = null)
             => gen_math_ops.add_v2(x, y, name);
@@ -182,15 +168,12 @@ namespace Tensorflow
         }
 
         public static Tensor cumsum<T>(Tensor x, T axis = default, bool exclusive = false, bool reverse = false, string name = null)
-        {
-            return tf_with(ops.name_scope(name, "Cumsum", new { x }), scope =>
-              {
-                  name = scope;
-                  x = ops.convert_to_tensor(x, name: "x");
-
-                  return gen_math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name);
-              });
-        }
+            => tf_with(ops.name_scope(name, "Cumsum", new { x }), scope =>
+            {
+                name = scope;
+                return tf.Context.ExecuteOp("Cumsum", name, new ExecuteOpArgs(x, axis)
+                    .SetAttributes(new { exclusive, reverse }));
+            });
 
         /// <summary>
         /// Computes Psi, the derivative of Lgamma (the log of the absolute value of
@@ -272,41 +255,13 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor erf(Tensor x, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("Erf", name, new { x }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Erf", name,
-                    null,
-                    x).FirstOrDefault(),
-                (op) =>
-                {
-                    var attrs = new object[]
-                    {
-                        "T", op.get_attr<TF_DataType>("T")
-                    };
-                    tf.Runner.RecordGradient("Erf", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(x));
+            => tf.Context.ExecuteOp("Erf", name, new ExecuteOpArgs(x));
 
         public static Tensor sqrt(Tensor x, string name = null)
             => gen_math_ops.sqrt(x, name: name);
 
         public static Tensor multiply(Tensor x, Tensor y, string name = null)
-            => tf.Context.RunInAutoMode2(
-                () => tf.OpDefLib._apply_op_helper("Mul", name, new { x, y }).output,
-                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Mul", name,
-                    null,
-                    x, y).FirstOrDefault(),
-                (op) =>
-                {
-                    var attrs = new object[]
-                    {
-                        "T", op.get_attr<TF_DataType>("T")
-                    };
-                    tf.Runner.RecordGradient("Mul", op.inputs, attrs, op.outputs);
-                },
-                new Tensors(x, y));
+            => tf.Context.ExecuteOp("Mul", name, new ExecuteOpArgs(x, y));
 
         public static Tensor multiply<Tx, Ty>(Tx x, Ty y, string name = null)
             => gen_math_ops.mul(x, y, name: name);
@@ -753,23 +708,10 @@ namespace Tensorflow
             => tf_with(ops.name_scope(name, "Pow", new { x, y }), scope =>
             {
                 name = scope;
+                var x_tensor = ops.convert_to_tensor(x, name: "x");
+                var y_tensor = ops.convert_to_tensor(y, name: "y", dtype: x_tensor.dtype.as_base_dtype());
 
-                if (tf.executing_eagerly())
-                {
-                    var x_tensor = ops.convert_to_tensor(x, name: "x");
-                    var y_tensor = ops.convert_to_tensor(y, name: "y", dtype: x_tensor.dtype.as_base_dtype());
-
-                    var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                        "Pow", name,
-                        null,
-                        x_tensor, y_tensor);
-
-                    return results[0];
-                }
-
-                var _op = tf.OpDefLib._apply_op_helper("Pow", name, args: new { x, y });
-
-                return _op.output;
+                return tf.Context.ExecuteOp("Pow", name, new ExecuteOpArgs(x_tensor, y_tensor));
             });
 
         public static Tensor range(object start, object limit = null, object delta = null, TF_DataType dtype = TF_DataType.DtInvalid, string name = "range")
@@ -851,21 +793,41 @@ namespace Tensorflow
         public static Tensor batch_matmul(Tensor x, Tensor y,
             bool adj_x = false, bool adj_y = false,
             string name = null)
-        {
-            Tensor result = null;
-
-            tf_with(ops.name_scope(name, "MatMul", new Tensor[] { x, y }), scope =>
+            => tf_with(ops.name_scope(name, "MatMul", new Tensor[] { x, y }), scope =>
             {
                 name = scope;
 
                 x = ops.convert_to_tensor(x, name: "a");
                 y = ops.convert_to_tensor(y, name: "b");
 
-                result = gen_math_ops.batch_mat_mul(x, y, adj_x, adj_y, name);
+                return tf.Context.ExecuteOp("BatchMatMul", name, new ExecuteOpArgs(x, y)
+                    .SetAttributes(new { adj_x, adj_y }));
             });
 
-            return result;
-        }
+        public static Tensor bincount(Tensor arr, Tensor weights = null,
+             Tensor minlength = null,
+             Tensor maxlength = null,
+             TF_DataType dtype = TF_DataType.TF_INT32,
+             string name = null,
+             TensorShape axis = null,
+             bool binary_output = false)
+            => tf_with(ops.name_scope(name, "bincount"), scope =>
+            {
+                name = scope;
+                if(!binary_output && axis == null)
+                {
+                    var array_is_nonempty = math_ops.reduce_prod(array_ops.shape(arr)) > 0;
+                    var output_size = math_ops.cast(array_is_nonempty, dtypes.int32) * (math_ops.reduce_max(arr) + 1);
+                    if (minlength != null)
+                        output_size = math_ops.maximum(minlength, output_size);
+                    if (maxlength != null)
+                        output_size = math_ops.minimum(maxlength, output_size);
+                    var weights = constant_op.constant(new long[0], dtype: dtype);
+                    return tf.Context.ExecuteOp("Bincount", name, new ExecuteOpArgs(arr, output_size, weights));
+                }
+
+                throw new NotImplementedException("");
+            });
 
         /// <summary>
         /// Returns the complex conjugate of a complex number.

src/TensorFlowNET.Core/Operations/string_ops.cs

@@ -14,12 +14,22 @@
    limitations under the License.
 ******************************************************************************/
 
+using NumSharp;
+using Tensorflow.Framework;
 using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
     public class string_ops
     {
+        public Tensor lower(Tensor input, string encoding = "", string name = null)
+            => tf.Context.ExecuteOp("StringLower", name, new ExecuteOpArgs(input, encoding));
+
+        public Tensor regex_replace(Tensor input, string pattern, string rewrite,
+                bool replace_global = true, string name = null)
+                    => tf.Context.ExecuteOp("StaticRegexReplace", name, new ExecuteOpArgs(input)
+                        .SetAttributes(new { pattern, rewrite, replace_global }));
+        
         /// <summary>
         /// Return substrings from `Tensor` of strings.
         /// </summary>
@@ -31,28 +41,93 @@ namespace Tensorflow
         /// <returns></returns>
         public Tensor substr<T>(T input, int pos, int len,
                 string @uint = "BYTE", string name = null)
+            => tf.Context.ExecuteOp("Substr", name, new ExecuteOpArgs(input, pos, len)
+                .SetAttributes(new { unit = @uint }));
+
+        /// <summary>
+        /// Computes the length of each string given in the input tensor.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <param name="name"></param>
+        /// <param name="unit"></param>
+        /// <returns></returns>
+        public Tensor string_length(Tensor input, string name = null, string unit = "BYTE")
+            => tf.Context.ExecuteOp("StringLength", name, new ExecuteOpArgs(input)
+            {
+                GetGradientAttrs = op => new
+                {
+                    unit = op.get_attr<string>("unit")
+                }
+            }.SetAttributes(new { unit }));
+
+        public RaggedTensor string_split_v2(Tensor input, string sep = "", int maxsplit = -1, string name = null)
         {
-            if (tf.Context.executing_eagerly())
+            return tf_with(ops.name_scope(name, "StringSplit"), scope =>
             {
-                var input_tensor = tf.constant(input);
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Substr", name,
-                    null,
-                    input, pos, len,
-                    "unit", @uint);
-
-                return results[0];
-            }
+                var sep_tensor = ops.convert_to_tensor(sep, dtype: TF_DataType.TF_STRING);
+                var result = tf.Context.ExecuteOp("StringSplitV2", name,
+                    new ExecuteOpArgs(input, sep)
+                    {
+                        GetGradientAttrs = op => new
+                        {
+                            maxsplit = op.get_attr<int>("maxsplit")
+                        }
+                    }.SetAttributes(new { maxsplit }));
+                var (indices, values, shape) = (result[0], result[1], result[2]);
+                indices.set_shape(new TensorShape(-1, 2));
+                values.set_shape(new TensorShape(-1));
+                shape.set_shape(new TensorShape(2));
 
-            var _op = tf.OpDefLib._apply_op_helper("Substr", name: name, args: new
+                var sparse_result = new SparseTensor(indices, values, shape);
+                return RaggedTensor.from_value_rowids(sparse_result.values,
+                    value_rowids: sparse_result.indices[Slice.All, 0],
+                    nrows: sparse_result.dense_shape[0],
+                    validate: false);
+            });
+        }
+
+        public (RaggedTensor, RaggedTensor) unicode_decode_with_offsets(Tensor input, string input_encoding, string errors,
+            int replacement_char = 0xFFFD, bool replace_control_characters = false, string name = null)
+        {
+            return tf_with(ops.name_scope(name, "UnicodeDecodeWithOffsets"), scope =>
             {
-                input,
-                pos,
-                len,
-                unit = @uint
+                var (codepoints, byte_start_offsets) = _unicode_decode(input, input_encoding, errors, 
+                    replacement_char, replace_control_characters,
+                    with_offsets: true, name: name);
+                return (codepoints, byte_start_offsets);
             });
+        }
+
+        (RaggedTensor, RaggedTensor) _unicode_decode(Tensor input, string input_encoding, string errors, int replacement_char,
+                    bool replace_control_characters, bool with_offsets, string name = null)
+        {
+            if (with_offsets)
+            {
+                var flat_result = tf.Context.ExecuteOp("UnicodeDecodeWithOffsets", name, new ExecuteOpArgs(input)
+                {
+                    GetGradientAttrs = op => new
+                    {
+                        input_encoding = op.get_attr<string>("input_encoding"),
+                        errors = op.get_attr<string>("errors"),
+                        replacement_char = op.get_attr<int>("replacement_char"),
+                        replace_control_characters = op.get_attr<bool>("replace_control_characters"),
+                        Tsplits = op.get_attr<TF_DataType>("Tsplits")
+                    }
+                }.SetAttributes(new
+                {
+                    input_encoding,
+                    errors,
+                    replacement_char,
+                    replace_control_characters
+                }));
+
+                var codepoints = RaggedTensor.from_row_splits(flat_result[1], flat_result[0], validate: false);
+
+                var offsets = RaggedTensor.from_row_splits(flat_result[2], flat_result[0], validate: false);
+                return (codepoints, offsets);
+            }
 
-            return _op.output;
+            return (null, null);
         }
     }
 }

src/TensorFlowNET.Core/Tensorflow.Binding.csproj

@@ -5,7 +5,7 @@
     <AssemblyName>TensorFlow.NET</AssemblyName>
     <RootNamespace>Tensorflow</RootNamespace>
     <TargetTensorFlow>2.2.0</TargetTensorFlow>
-    <Version>0.33.0</Version>
+    <Version>0.40.0</Version>
     <LangVersion>8.0</LangVersion>
     <Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
     <Company>SciSharp STACK</Company>
@@ -19,7 +19,7 @@
     <Description>Google's TensorFlow full binding in .NET Standard.
 Building, training and infering deep learning models.
 https://tensorflownet.readthedocs.io</Description>
-    <AssemblyVersion>0.33.0.0</AssemblyVersion>
+    <AssemblyVersion>0.40.0.0</AssemblyVersion>
     <PackageReleaseNotes>tf.net 0.20.x and above are based on tensorflow native 2.x.
 
 * Eager Mode is added finally.
@@ -29,8 +29,10 @@ https://tensorflownet.readthedocs.io</Description>
 * Improve memory usage.
 
 TensorFlow .NET v0.3x is focused on making more Keras API works.
-Keras API is a separate package released as TensorFlow.Keras.</PackageReleaseNotes>
-    <FileVersion>0.33.0.0</FileVersion>
+Keras API is a separate package released as TensorFlow.Keras.
+
+tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.</PackageReleaseNotes>
+    <FileVersion>0.40.0.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
     <SignAssembly>true</SignAssembly>
@@ -48,6 +50,7 @@ Keras API is a separate package released as TensorFlow.Keras.</PackageReleaseNot
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>TRACE;DEBUG</DefineConstants>
     <PlatformTarget>x64</PlatformTarget>
+    <DocumentationFile>TensorFlow.NET.xml</DocumentationFile>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
@@ -84,7 +87,7 @@ Keras API is a separate package released as TensorFlow.Keras.</PackageReleaseNot
   <ItemGroup>
     <PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.138" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.1" />
-    <PackageReference Include="NumSharp.Lite" Version="0.1.12" />
+    <PackageReference Include="NumSharp" Version="0.30.0" />
     <PackageReference Include="Protobuf.Text" Version="0.5.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
   </ItemGroup>

src/TensorFlowNET.Core/Tensors/EagerTensorV2.cs

@@ -7,7 +7,7 @@ using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
-    public class EagerTensorV2 : DisposableObject, ITensor
+    public class EagerTensorV2 : DisposableObject
     {
         SafeTensorHandleHandle EagerTensorHandle;
         public string Device

src/TensorFlowNET.Core/Tensors/ITensor.cs

@@ -1,7 +0,0 @@
-namespace Tensorflow
-{
-    public interface ITensor
-    {
-
-    }
-}

src/TensorFlowNET.Core/Tensors/Ragged/RaggedTensor.cs

@@ -0,0 +1,147 @@
+/*****************************************************************************
+   Copyright 2021 Haiping Chen. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Linq;
+using Tensorflow.Framework;
+using static Tensorflow.Binding;
+using NumSharp;
+
+namespace Tensorflow
+{
+    /// <summary>
+    /// Represents a ragged tensor.
+    /// </summary>
+    public class RaggedTensor : CompositeTensor
+    {
+        Tensor _values;
+        RowPartition _row_partition;
+        Tensor _row_splits => _row_partition.row_splits;
+
+        public TF_DataType dtype => _values.dtype;
+        public TensorShape shape
+        {
+            get
+            {
+                var nrows = _row_partition.static_nrows;
+                var ncols = _row_partition.static_uniform_row_length;
+                return new TensorShape(nrows, ncols);
+            }
+        }
+
+        public RaggedTensor this[params Slice[] slices]
+        {
+            get
+            {
+                var row_key = slices[0];
+                var inner_keys = slices.Skip(1).ToArray();
+
+                var args = tensor_util.ParseSlices(slices);
+
+                return tf_with(ops.name_scope(null, "RaggedGetItem", args), scope =>
+                {
+                    string name = scope;
+                    return _ragged_getitem_inner_dimensions(this, inner_keys);
+                });
+            }
+        }
+
+        RaggedTensor _ragged_getitem_inner_dimensions(RaggedTensor input, Slice[] slices)
+        {
+            return input;
+        }
+
+        public RaggedTensor(Tensor values,
+            bool @internal = true,
+            RowPartition row_partition = null)
+        {
+            _values = values;
+            _row_partition = row_partition;
+        }
+
+        public static RaggedTensor from_row_partition(Tensor values, RowPartition row_partition, bool validate = true)
+        {
+            return new RaggedTensor(values, @internal: true, row_partition: row_partition);
+        }
+
+        /// <summary>
+        /// Creates a `RaggedTensor` with rows partitioned by `value_rowids`.
+        /// </summary>
+        /// <param name="values"></param>
+        /// <param name="value_rowids"></param>
+        /// <param name="nrows"></param>
+        /// <param name="name"></param>
+        /// <param name="validate"></param>
+        /// <returns></returns>
+        public static RaggedTensor from_value_rowids(Tensor values, Tensor value_rowids, 
+            Tensor nrows = null, string name = null, bool validate = true)
+        {
+            return tf_with(ops.name_scope(name, "RaggedFromValueRowIds"), scope =>
+            {
+                var row_partition = RowPartition.from_value_rowids(value_rowids,
+                  nrows: nrows,
+                  validate: validate);
+                return from_row_partition(values, row_partition, validate: validate);
+            });
+        }
+
+        public static RaggedTensor from_row_splits(Tensor values, Tensor row_splits, 
+            string name = null, bool validate = true)
+        {
+            return tf_with(ops.name_scope(name, "RaggedFromRowSplits"), scope =>
+            {
+                var row_partition = RowPartition.from_row_splits(row_splits,
+                  validate: validate);
+                return from_row_partition(values, row_partition, validate: validate);
+            });
+        }
+
+        Tensor _to_variant(bool batched_input = false, string name = null)
+            => tf_with(ops.name_scope(name, "RaggedToVariant"), scope =>
+            {
+                return tf.Context.ExecuteOp("RaggedTensorToVariant", name,
+                    new ExecuteOpArgs(nested_row_splits, flat_values)
+                    {
+                        GetGradientAttrs = op => new
+                        {
+                            RAGGED_RANK = op.get_attr<int>("RAGGED_RANK"),
+                            Tvalues = op.get_attr<TF_DataType>("Tvalues"),
+                            Tsplits = op.get_attr<TF_DataType>("Tsplits"),
+                            batched_input = op.get_attr<bool>("batched_input")
+                        }
+                    }.SetAttributes(new { batched_input }));
+            });
+
+        Tensor flat_values
+            => _values;
+
+        Tensor[] nested_row_splits
+            => new[] { _row_splits };
+
+        public override string ToString()
+            => $"tf.RaggedTensor: shape={shape} [{string.Join(", ", _values.StringData().Take(10))}]";
+
+        public static implicit operator Tensor(RaggedTensor indexedSlices)
+            => indexedSlices._to_variant();
+
+        public static implicit operator RaggedTensor(Tensor tensor)
+        {
+            return tensor.Tag as RaggedTensor;
+        }
+    }
+}

src/TensorFlowNET.Core/Tensors/Ragged/RowPartition.cs

@@ -0,0 +1,103 @@
+/*****************************************************************************
+   Copyright 2021 Haiping Chen. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Framework;
+using static Tensorflow.Binding;
+
+namespace Tensorflow
+{
+    /// <summary>
+    /// Partitioning of a sequence of values into contiguous subsequences ("rows").
+    /// </summary>
+    public class RowPartition : CompositeTensor
+    {
+        Tensor _row_splits;
+        public Tensor row_splits => _row_splits;
+        Tensor _row_lengths;
+        Tensor _value_rowids;
+        Tensor _nrows;
+
+        public int static_nrows
+        {
+            get
+            {
+                return _row_splits.shape[0] - 1;
+            }
+        }
+
+        public int static_uniform_row_length
+        {
+            get
+            {
+                return -1;
+            }
+        }
+
+        public RowPartition(Tensor row_splits, 
+            Tensor row_lengths = null, Tensor value_rowids = null, Tensor nrows = null,
+            Tensor uniform_row_length = null)
+        {
+            _row_splits = row_splits;
+            _row_lengths = row_lengths;
+            _value_rowids = value_rowids;
+            _nrows = nrows;
+        }
+
+        /// <summary>
+        /// Creates a `RowPartition` with rows partitioned by `value_rowids`.
+        /// </summary>
+        /// <param name="value_rowids"></param>
+        /// <param name="nrows"></param>
+        /// <param name="validate"></param>
+        /// <param name="preferred_dtype"></param>
+        /// <returns></returns>
+        public static RowPartition from_value_rowids(Tensor value_rowids,
+            Tensor nrows = null, bool validate = true, TF_DataType preferred_dtype = TF_DataType.DtInvalid)
+        {
+            return tf_with(ops.name_scope(null, "RowPartitionFromValueRowIds"), scope =>
+            {
+                var value_rowids_int32 = math_ops.cast(value_rowids, dtypes.int32);
+                var nrows_int32 = math_ops.cast(nrows, dtypes.int32);
+                var row_lengths = tf.math.bincount(value_rowids_int32, 
+                    minlength: nrows_int32,
+                    maxlength: nrows_int32,
+                    dtype: value_rowids.dtype);
+                var row_splits = array_ops.concat(new object[]
+                {
+                    ops.convert_to_tensor(new long[] { 0 }),
+                    tf.cumsum(row_lengths)
+                }, axis: 0);
+
+                return new RowPartition(row_splits,
+                    row_lengths: row_lengths,
+                    value_rowids: value_rowids,
+                    nrows: nrows);
+            });
+        }
+
+        public static RowPartition from_row_splits(Tensor row_splits,
+            bool validate = true, TF_DataType preferred_dtype = TF_DataType.DtInvalid)
+        {
+            return tf_with(ops.name_scope(null, "RowPartitionFromRowSplits"), scope =>
+            {
+                return new RowPartition(row_splits);
+            });
+        }
+    }
+}

src/TensorFlowNET.Core/Tensors/Ragged/SparseTensor.cs

@@ -0,0 +1,76 @@
+/*****************************************************************************
+   Copyright 2021 Haiping Chen. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Linq;
+using Tensorflow.Framework;
+using static Tensorflow.Binding;
+
+namespace Tensorflow
+{
+    /// <summary>
+    /// Represents a sparse tensor.
+    /// </summary>
+    public class SparseTensor : CompositeTensor
+    {
+        public Tensor indices;
+
+        public Tensor values;
+
+        public Tensor dense_shape;
+
+        public SparseTensor(Tensor indices, Tensor values, Tensor dense_shape)
+        {
+            this.indices = indices;
+            this.values = values;
+            this.dense_shape = dense_shape;
+            _init();
+        }
+
+        public SparseTensor(long[,] indices_, Array values_, long[] dense_shape_)
+        {
+            tf_with(ops.name_scope(null, "SparseTensor", new { }), delegate
+            {
+                indices = ops.convert_to_tensor(
+                    indices_, name: "indices", dtype: dtypes.int64);
+                values = ops.convert_to_tensor(values_, name: "values");
+                dense_shape = ops.convert_to_tensor(
+                    dense_shape_, name: "dense_shape", dtype: dtypes.int64);
+            });
+            _init();
+        }
+
+        void _init()
+        {
+            var indices_shape = indices.TensorShape.with_rank(2);
+            var values_shape = values.TensorShape.with_rank(1);
+            var dense_shape_shape = dense_shape.TensorShape.with_rank(1);
+
+            indices_shape["0"].merge_with(values_shape[0]);
+            indices_shape["1"].merge_with(dense_shape_shape[0]);
+        }
+
+        public static implicit operator Tensor(SparseTensor indexedSlices)
+        {
+            return indexedSlices.values;
+        }
+
+        public static implicit operator SparseTensor(Tensor tensor)
+        {
+            return tensor.Tag as SparseTensor;
+        }
+    }
+}

src/TensorFlowNET.Core/Tensors/Tensor.Index.cs

@@ -60,13 +60,9 @@ namespace Tensorflow
             }
         }
 
-        public Tensor this[Range slices]
-            => throw new NotImplementedException("");
-
         public Tensor this[params string[] slices]
             => this[slices.Select(x => new Slice(x)).ToArray()];
 
-
         public Tensor slice(Slice slice)
         {
             var slice_spec = new int[] { slice.Start.Value };

src/TensorFlowNET.Core/Tensors/Tensor.String.cs

@@ -8,27 +8,7 @@ namespace Tensorflow
 {
     public partial class Tensor
     {
-        const ulong TF_TSRING_SIZE = 24;
-
-        public IntPtr StringTensor25(string[] strings, TensorShape shape)
-        {
-            var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
-                shape.dims.Select(x => (long)x).ToArray(),
-                shape.ndim,
-                (ulong)shape.size * TF_TSRING_SIZE);
-
-            var data = c_api.TF_TensorData(handle);
-            var tstr = c_api.TF_StringInit(handle);
-            // AllocationHandle = tstr;
-            // AllocationType = AllocationType.Tensorflow;
-            for (int i = 0; i< strings.Length; i++)
-            {
-                c_api.TF_StringCopy(tstr, strings[i], strings[i].Length);
-                tstr += (int)TF_TSRING_SIZE;
-            }
-            // c_api.TF_StringDealloc(tstr);
-            return handle;
-        }
+        const int TF_TSRING_SIZE = 24;
 
         public IntPtr StringTensor(string[] strings, TensorShape shape)
         {
@@ -40,69 +20,28 @@ namespace Tensorflow
             return StringTensor(buffer, shape);
         }
 
-        public unsafe IntPtr StringTensor(byte[][] buffer, TensorShape shape)
+        public IntPtr StringTensor(byte[][] buffer, TensorShape shape)
         {
-            ulong size = 0;
-            foreach (var b in buffer)
-                size += c_api.TF_StringEncodedSize((ulong)b.Length);
-
-            var src_size = size + (ulong)buffer.Length * sizeof(ulong);
             var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
-                shape.dims.Select(x => (long)x).ToArray(),
+                shape.ndim == 0 ? null : shape.dims.Select(x => (long)x).ToArray(),
                 shape.ndim,
-                src_size);
-            AllocationType = AllocationType.Tensorflow;
+                (ulong)shape.size * TF_TSRING_SIZE);
 
-            IntPtr data_start = c_api.TF_TensorData(handle);
-            IntPtr string_start = data_start + buffer.Length * sizeof(ulong);
-            IntPtr limit = data_start + (int)src_size;
-            ulong offset = 0;
+            var tstr = c_api.TF_TensorData(handle);
+#if TRACK_TENSOR_LIFE
+            print($"New TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstr.ToString("x16")}");
+#endif
             for (int i = 0; i < buffer.Length; i++)
             {
-                Marshal.WriteInt64(data_start, i * sizeof(ulong), (long)offset);
-                if (buffer[i].Length == 0)
-                {
-                    Marshal.WriteByte(string_start, 0);
-                    break;
-                }
-
-                fixed (byte* src = &buffer[i][0])
-                {
-                    /*Marshal.WriteByte(string_start, Convert.ToByte(buffer[i].Length));
-                    tf.memcpy((string_start + 1).ToPointer(), src, (ulong)buffer[i].Length);
-                    string_start += buffer[i].Length + 1;
-                    offset += buffer[i].Length + 1;*/
-
-                    var written = c_api.TF_StringEncode(src, (ulong)buffer[i].Length, (byte*)string_start, (ulong)(limit.ToInt64() - string_start.ToInt64()), tf.Status.Handle);
-                    tf.Status.Check(true);
-                    string_start += (int)written;
-                    offset += written;
-                }
+                c_api.TF_StringInit(tstr);
+                c_api.TF_StringCopy(tstr, buffer[i], buffer[i].Length);
+                var data = c_api.TF_StringGetDataPointer(tstr);
+                tstr += TF_TSRING_SIZE;
             }
 
             return handle;
         }
 
-        public string[] StringData25()
-        {
-            string[] strings = new string[c_api.TF_Dim(_handle, 0)];
-            var tstrings = TensorDataPointer; 
-            for (int i = 0; i< strings.Length; i++)
-            {
-                var tstringData = c_api.TF_StringGetDataPointer(tstrings);
-                /*var size = c_api.TF_StringGetSize(tstrings);
-                var capacity = c_api.TF_StringGetCapacity(tstrings);
-                var type = c_api.TF_StringGetType(tstrings);*/
-                strings[i] = c_api.StringPiece(tstringData);
-                tstrings += (int)TF_TSRING_SIZE;
-            }
-            return strings;
-        }
-
-        /// <summary>
-        ///     Extracts string array from current Tensor.
-        /// </summary>
-        /// <exception cref="InvalidOperationException">When <see cref="dtype"/> != TF_DataType.TF_STRING</exception>
         public string[] StringData()
         {
             var buffer = StringBytes();
@@ -114,7 +53,7 @@ namespace Tensorflow
             return _str;
         }
 
-        public unsafe byte[][] StringBytes()
+        public byte[][] StringBytes()
         {
             if (dtype != TF_DataType.TF_STRING)
                 throw new InvalidOperationException($"Unable to call StringData when dtype != TF_DataType.TF_STRING (dtype is {dtype})");
@@ -123,24 +62,22 @@ namespace Tensorflow
             // TF_STRING tensors are encoded with a table of 8-byte offsets followed by TF_StringEncode-encoded bytes.
             // [offset1, offset2,...,offsetn, s1size, s1bytes, s2size, s2bytes,...,snsize,snbytes]
             //
-            long size = 1;
+            int size = 1;
             foreach (var s in TensorShape.dims)
                 size *= s;
 
             var buffer = new byte[size][];
-            var data_start = c_api.TF_TensorData(_handle);
-            data_start += (int)(size * sizeof(ulong));
+            var tstrings = TensorDataPointer;
             for (int i = 0; i < buffer.Length; i++)
             {
-                IntPtr dst = IntPtr.Zero;
-                ulong dstLen = 0;
-                var read = c_api.TF_StringDecode((byte*)data_start, bytesize, (byte**)&dst, ref dstLen, tf.Status.Handle);
-                tf.Status.Check(true);
-                buffer[i] = new byte[(int)dstLen];
-                Marshal.Copy(dst, buffer[i], 0, buffer[i].Length);
-                data_start += (int)read;
+                var data = c_api.TF_StringGetDataPointer(tstrings);
+                var len = c_api.TF_StringGetSize(tstrings);
+                buffer[i] = new byte[len];
+                // var capacity = c_api.TF_StringGetCapacity(tstrings);
+                // var type = c_api.TF_StringGetType(tstrings);
+                Marshal.Copy(data, buffer[i], 0, Convert.ToInt32(len));
+                tstrings += TF_TSRING_SIZE;
             }
-
             return buffer;
         }
     }

src/TensorFlowNET.Core/Tensors/Tensor.cs

@@ -15,7 +15,6 @@
 ******************************************************************************/
 
 using NumSharp;
-using NumSharp.Backends.Unmanaged;
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
@@ -24,7 +23,6 @@ using System.Runtime.InteropServices;
 using Tensorflow.Eager;
 using Tensorflow.Framework;
 using Tensorflow.Keras.Engine;
-using Tensorflow.Variables;
 using static Tensorflow.Binding;
 
 namespace Tensorflow
@@ -35,9 +33,7 @@ namespace Tensorflow
     /// </summary>
     [SuppressMessage("ReSharper", "ConvertToAutoProperty")]
     public partial class Tensor : DisposableObject,
-        ITensor,
         ITensorOrOperation,
-        _TensorLike,
         ITensorOrTensorArray,
         IPackable<Tensor>,
         ICanBeFlattened
@@ -99,6 +95,7 @@ namespace Tensorflow
         public SafeTensorHandleHandle EagerTensorHandle { get; set; }
 
         public bool IsEagerTensor => this is EagerTensor;
+        public bool IsSparseTensor => this is SparseTensor;
 
         /// <summary>
         ///     Returns the shape of a tensor.
@@ -287,6 +284,22 @@ namespace Tensorflow
                     throw new InvalidOperationException($"Tensor.AllocationHandle is not null ({AllocationHandle}) but AllocationType is not matched to a C# allocation type ({AllocationType}).");
             }
 
+            if (dtype == TF_DataType.TF_STRING)
+            {
+                int size = 1;
+                foreach (var s in TensorShape.dims)
+                    size *= s;
+                var tstr = TensorDataPointer;
+#if TRACK_TENSOR_LIFE
+                print($"Delete TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstrings.ToString("x16")}");
+#endif
+                for (int i = 0; i < size; i++)
+                {
+                    c_api.TF_StringDealloc(tstr);
+                    tstr += TF_TSRING_SIZE;
+                }
+            }
+
             c_api.TF_DeleteTensor(handle);
         }
 

src/TensorFlowNET.Core/Tensors/c_api.tensor.cs

@@ -182,7 +182,10 @@ namespace Tensorflow
         public static extern unsafe ulong TF_StringEncode(byte* src, ulong src_len, byte* dst, ulong dst_len, SafeStatusHandle status);
 
         [DllImport(TensorFlowLibName)]
-        public static extern IntPtr TF_StringInit(IntPtr t);
+        public static extern void TF_StringInit(IntPtr t);
+
+        [DllImport(TensorFlowLibName)]
+        public static extern void TF_StringCopy(IntPtr dst, byte[] text, long size);
 
         [DllImport(TensorFlowLibName)]
         public static extern void TF_StringCopy(IntPtr dst, string text, long size);

src/TensorFlowNET.Core/Training/gen_training_ops.cs

@@ -21,46 +21,19 @@ namespace Tensorflow
 {
     public class gen_training_ops
     {
-        public static Operation resource_apply_adam(Tensor var, Tensor m, Tensor v, Tensor beta1_power, Tensor beta2_power,
+        public static Tensor resource_apply_adam(Tensor var, Tensor m, Tensor v, Tensor beta1_power, Tensor beta2_power,
             Tensor lr, Tensor beta1, Tensor beta2, Tensor epsilon, Tensor grad,
             bool use_locking = false, bool use_nesterov = false, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var result = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResourceApplyAdam", name,
-                    null,
-                    var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
-                    "use_locking", use_locking,
-                    "use_nesterov", use_nesterov);
-                return null;
-            }
-
-            throw new NotImplementedException("");
-        }
+                => tf.Context.ExecuteOp("ResourceApplyAdam", name, 
+                    new ExecuteOpArgs(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad)
+                        .SetAttributes(new { use_locking, use_nesterov }));
 
         public static Tensor apply_adam(Tensor var, Tensor m, Tensor v, Tensor beta1_power, Tensor beta2_power,
             Tensor lr, Tensor beta1, Tensor beta2, Tensor epsilon, Tensor grad,
             bool use_locking = false, bool use_nesterov = false, string name = null)
-        {
-            var _op = tf.OpDefLib._apply_op_helper("ApplyAdam", name, new
-            {
-                var,
-                m,
-                v,
-                beta1_power,
-                beta2_power,
-                lr,
-                beta1,
-                beta2,
-                epsilon,
-                grad,
-                use_locking,
-                use_nesterov
-            });
-
-            return _op.outputs[0];
-        }
+                => tf.Context.ExecuteOp("ApplyAdam", name, 
+                    new ExecuteOpArgs(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad)
+                        .SetAttributes(new { use_locking, use_nesterov }));
 
         public static Tensor apply_gradient_descent(IVariableV1 var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null)
         {
@@ -75,27 +48,8 @@ namespace Tensorflow
             return _op.output;
         }
 
-        public static Operation resource_apply_gradient_descent(Tensor var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var result = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "ResourceApplyGradientDescent", name,
-                    null,
-                    var, alpha, delta,
-                    "use_locking", use_locking);
-                return null;
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("ResourceApplyGradientDescent", name, new
-            {
-                var,
-                alpha,
-                delta,
-                use_locking
-            });
-
-            return _op;
-        }
+        public static Tensor resource_apply_gradient_descent(Tensor var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null)
+            => tf.Context.ExecuteOp("ResourceApplyGradientDescent", name, 
+                new ExecuteOpArgs(var, alpha, delta).SetAttributes(new { use_locking }));
     }
 }

src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs

@@ -59,31 +59,8 @@ namespace Tensorflow
             bool validate_shape = true,
             bool use_locking = true,
             string name = null)
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Assign", name,
-                    null,
-                    @ref, value,
-                    "validate_shape", validate_shape,
-                    "use_locking", use_locking);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Assign", name: name, args: new { @ref, value, validate_shape, use_locking });
-
-            var _result = _op.outputs;
-            var _inputs_flat = _op.inputs;
-
-            var _attrs = new Dictionary<string, object>();
-            _attrs["T"] = _op.get_attr("T");
-            _attrs["validate_shape"] = _op.get_attr("validate_shape");
-            _attrs["use_locking"] = _op.get_attr("use_locking");
-
-            return _result[0];
-        }
+                => tf.Context.ExecuteOp("Assign", name, new ExecuteOpArgs(@ref, value)
+                    .SetAttributes(new { validate_shape, use_locking }));
 
         public static Tensor assign_add<T>(IVariableV1 @ref, T value, bool use_locking = false, string name = null)
         {

src/TensorFlowNET.Keras/Activations/Activations.Relu.cs

@@ -4,21 +4,7 @@ namespace Tensorflow.Keras
 {
     public partial class Activations
     {
-        public Activation Relu = (features, name) =>
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Relu", name,
-                    null,
-                    features);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Relu", name: name, args: new { features });
-
-            return _op.output;
-        };
+        public Activation Relu = (features, name)
+            => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features));
     }
 }

src/TensorFlowNET.Keras/Activations/Activations.Sigmoid.cs

@@ -5,21 +5,7 @@ namespace Tensorflow.Keras
 {
     public partial class Activations
     {
-        public Activation Sigmoid = (features, name) =>
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Sigmoid", name,
-                    null,
-                    features);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Sigmoid", name: name, args: new { x = features });
-
-            return _op.output;
-        };
+        public Activation Sigmoid = (features, name) 
+            => tf.Context.ExecuteOp("Sigmoid", name, new ExecuteOpArgs(features));
     }
 }

src/TensorFlowNET.Keras/Activations/Activations.Tanh.cs

@@ -5,21 +5,7 @@ namespace Tensorflow.Keras
 {
     public partial class Activations
     {
-        public Activation Tanh = (features, name) =>
-        {
-            if (tf.executing_eagerly())
-            {
-                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
-                    "Tanh", name,
-                    null,
-                    features);
-
-                return results[0];
-            }
-
-            var _op = tf.OpDefLib._apply_op_helper("Tanh", name: name, args: new { x = features });
-
-            return _op.output;
-        };
+        public Activation Tanh = (features, name) 
+            => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features));
     }
 }

src/TensorFlowNET.Keras/Datasets/MNIST.cs

@@ -45,8 +45,8 @@ namespace Tensorflow.Keras.Datasets
 
         (NDArray, NDArray) LoadX(byte[] bytes)
         {
-            var y = np.Load_Npz<byte[,,]>(bytes);
-            return (y["x_train.npy"], y["x_test.npy"]);
+            var x = np.Load_Npz<byte[,,]>(bytes);
+            return (x["x_train.npy"], x["x_test.npy"]);
         }
 
         (NDArray, NDArray) LoadY(byte[] bytes)

src/TensorFlowNET.Keras/Engine/CombinerPreprocessingLayer.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+
+namespace Tensorflow.Keras.Engine
+{
+    public class CombinerPreprocessingLayer : Layer
+    {
+        PreprocessingLayerArgs args;
+        protected ICombiner combiner;
+        protected bool _previously_updated;
+
+        public CombinerPreprocessingLayer(PreprocessingLayerArgs args)
+            : base(args)
+        {
+            _previously_updated = false;
+        }
+
+        public virtual void adapt(IDatasetV2 data, bool reset_state = true)
+        {
+            IAccumulator accumulator;
+            if (!reset_state)
+                accumulator = combiner.Restore();
+
+            var next_data = data.make_one_shot_iterator();
+            var data_element = next_data.next();
+        }
+    }
+}

src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs

@@ -39,7 +39,7 @@ namespace Tensorflow.Keras.Engine.DataAdapters
             dataset = slice_inputs(indices_dataset, inputs);
         }
 
-        Tensor permutation(Tensor tensor)
+        Tensors permutation(Tensors tensor)
         {
             var indices = math_ops.range(num_samples, dtype: dtypes.int64);
             if (args.Shuffle)
@@ -82,7 +82,7 @@ namespace Tensorflow.Keras.Engine.DataAdapters
                     .Select(x => gen_array_ops.gather_v2(x, indices, 0))
                     .ToArray();
                 return new Tensors(results);
-            });
+            }, -1);
 
             return dataset.with_options(new DatasetOptions { });
         }

src/TensorFlowNET.Keras/Engine/Interfaces/IAccumulator.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Engine
+{
+    public interface IAccumulator
+    {
+    }
+}

src/TensorFlowNET.Keras/Engine/Interfaces/ICombiner.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Engine
+{
+    /// <summary>
+    /// Functional object that defines a shardable computation.
+    /// </summary>
+    public interface ICombiner
+    {
+        void Compute(Tensor values, IAccumulator accumulator = null);
+        void Merge();
+        void Extract();
+        IAccumulator Restore();
+        void Serialize();
+        void Deserialize();
+    }
+}

src/TensorFlowNET.Keras/Engine/MetricsContainer.cs

@@ -62,8 +62,8 @@ namespace Tensorflow.Keras.Engine
             {
                 var y_t_rank = y_t.rank;
                 var y_p_rank = y_p.rank;
-                var y_t_last_dim = y_t.shape[^1];
-                var y_p_last_dim = y_p.shape[^1];
+                var y_t_last_dim = y_t.shape[y_t.shape.Length - 1];
+                var y_p_last_dim = y_p.shape[y_p.shape.Length - 1];
 
                 bool is_binary = y_p_last_dim == 1;
                 bool is_sparse_categorical = (y_t_rank < y_p_rank || y_t_last_dim == 1) && y_p_last_dim > 1;

src/TensorFlowNET.Keras/Engine/Sequential.cs

@@ -14,6 +14,7 @@
    limitations under the License.
 ******************************************************************************/
 
+using System.Linq;
 using System.Collections.Generic;
 using Tensorflow.Keras.ArgsDefinition;
 using Tensorflow.Keras.Layers;
@@ -103,7 +104,7 @@ namespace Tensorflow.Keras.Engine
                 if (set_inputs)
                 {
                     // If an input layer (placeholder) is available.
-                    outputs = layer.InboundNodes[^1].Outputs;
+                    outputs = layer.InboundNodes.Last().Outputs;
                     inputs = layer_utils.get_source_inputs(outputs[0]);
                     built = true;
                     _has_explicit_input_shape = true;

src/TensorFlowNET.Keras/KerasInterface.cs

@@ -11,6 +11,7 @@ using Tensorflow.Keras.Metrics;
 using Tensorflow.Keras.Models;
 using Tensorflow.Keras.Optimizers;
 using Tensorflow.Keras.Saving;
+using Tensorflow.Keras.Utils;
 
 namespace Tensorflow.Keras
 {
@@ -27,6 +28,7 @@ namespace Tensorflow.Keras
         public OptimizerApi optimizers { get; } = new OptimizerApi();
         public MetricsApi metrics { get; } = new MetricsApi();
         public ModelsApi models { get; } = new ModelsApi();
+        public KerasUtils utils { get; } = new KerasUtils();
 
         public Sequential Sequential(List<ILayer> layers = null,
                 string name = null)
@@ -73,7 +75,7 @@ namespace Tensorflow.Keras
                 Tensor tensor = null)
         {
             if (batch_input_shape != null)
-                shape = batch_input_shape.dims[1..];
+                shape = batch_input_shape.dims.Skip(1).ToArray();
 
             var args = new InputLayerArgs
             {

src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs

@@ -42,7 +42,7 @@ namespace Tensorflow.Keras.Layers
             if (BatchInputShape != null)
             {
                 args.BatchSize = BatchInputShape.dims[0];
-                args.InputShape = BatchInputShape.dims[1..];
+                args.InputShape = BatchInputShape.dims.Skip(1).ToArray();
             }
 
             // moved to base class

src/TensorFlowNET.Keras/Layers/LayersApi.cs

@@ -9,6 +9,8 @@ namespace Tensorflow.Keras.Layers
 {
     public partial class LayersApi
     {
+        public Preprocessing preprocessing { get; } = new Preprocessing();
+
         /// <summary>
         /// Functional interface for the batch normalization layer.
         /// http://arxiv.org/abs/1502.03167
@@ -323,6 +325,16 @@ namespace Tensorflow.Keras.Layers
             return input_layer.InboundNodes[0].Outputs;
         }
 
+        public MaxPooling1D MaxPooling1D(int? pool_size = null,
+            int? strides = null,
+            string padding = "valid")
+            => new MaxPooling1D(new Pooling1DArgs
+            {
+                PoolSize = pool_size ?? 2,
+                Strides = strides ?? (pool_size ?? 2),
+                Padding = padding
+            });
+
         public MaxPooling2D MaxPooling2D(TensorShape pool_size = null,
             TensorShape strides = null,
             string padding = "valid")
@@ -446,6 +458,20 @@ namespace Tensorflow.Keras.Layers
         public GlobalAveragePooling2D GlobalAveragePooling2D()
             => new GlobalAveragePooling2D(new Pooling2DArgs { });
 
+        public GlobalAveragePooling1D GlobalAveragePooling1D(string data_format = "channels_last")
+            => new GlobalAveragePooling1D(new Pooling1DArgs { DataFormat = data_format });
+
+        public GlobalAveragePooling2D GlobalAveragePooling2D(string data_format = "channels_last")
+            => new GlobalAveragePooling2D(new Pooling2DArgs { DataFormat = data_format });
+
+        public GlobalMaxPooling1D GlobalMaxPooling1D(string data_format = "channels_last")
+            => new GlobalMaxPooling1D(new Pooling1DArgs { DataFormat = data_format });
+
+        public GlobalMaxPooling2D GlobalMaxPooling2D(string data_format = "channels_last")
+            => new GlobalMaxPooling2D(new Pooling2DArgs { DataFormat = data_format });
+
+
+
         Activation GetActivationByName(string name)
             => name switch
             {

src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class GlobalAveragePooling1D : GlobalPooling1D
+    {
+        public GlobalAveragePooling1D(Pooling1DArgs args)
+            : base(args)
+        {
+        }
+
+        protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null)
+        {
+            if (data_format == "channels_last")
+                return math_ops.reduce_mean(inputs, new int[] { 1 }, false);
+            else
+                return math_ops.reduce_mean(inputs, new int[] { 2 }, false);
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class GlobalMaxPooling1D : GlobalPooling1D
+    {
+        public GlobalMaxPooling1D(Pooling1DArgs args)
+            : base(args)
+        {
+        }
+
+        protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null)
+        {
+            if (data_format == "channels_last")
+                return math_ops.reduce_max(inputs, new int[] { 1 }, false);
+            else
+                return math_ops.reduce_max(inputs, new int[] { 2 }, false);
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class GlobalMaxPooling2D : GlobalPooling2D
+    {
+        public GlobalMaxPooling2D(Pooling2DArgs args)
+            : base(args)
+        {
+        }
+
+        protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null)
+        {
+            if (data_format == "channels_last")
+                return math_ops.reduce_max(inputs, new int[] { 1, 2 }, false);
+            else
+                return math_ops.reduce_max(inputs, new int[] { 2, 3 }, false);
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Pooling/GlobalPooling1D.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Utils;
+
+namespace Tensorflow.Keras.Layers
+{
+    public abstract class GlobalPooling1D : Layer
+    {
+        Pooling1DArgs args;
+        protected string data_format => args.DataFormat;
+        protected InputSpec input_spec;
+
+        public GlobalPooling1D(Pooling1DArgs args) : base(args)
+        {
+            this.args = args;
+            args.DataFormat = conv_utils.normalize_data_format(data_format);
+            input_spec = new InputSpec(ndim: 3);
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Pooling/MaxPooling1D.cs

@@ -0,0 +1,14 @@
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Operations;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class MaxPooling1D : Pooling1D
+    {
+        public MaxPooling1D(Pooling1DArgs args)
+            : base(args)
+        {
+            args.PoolFunction = new MaxPoolFunction();
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs

@@ -0,0 +1,62 @@
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Utils;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class Pooling1D : Layer
+    {
+        Pooling1DArgs args;
+        InputSpec input_spec;
+
+        public Pooling1D(Pooling1DArgs args)
+            : base(args)
+        {
+            this.args = args;
+            args.Padding = conv_utils.normalize_padding(args.Padding);
+            args.DataFormat = conv_utils.normalize_data_format(args.DataFormat);
+            input_spec = new InputSpec(ndim: 3);
+        }
+
+        protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null)
+        {
+            int[] pool_shape;
+            int[] strides;
+            if (args.DataFormat == "channels_last")
+            {
+                pool_shape = new int[] { 1, args.PoolSize, 1 };
+                strides = new int[] { 1, args.Strides, 1 };
+            }
+            else
+            {
+                pool_shape = new int[] { 1, 1, args.PoolSize };
+                strides = new int[] { 1, 1, args.Strides };
+            }
+
+            var outputs = args.PoolFunction.Apply(
+                inputs,
+                ksize: pool_shape,
+                strides: strides,
+                padding: args.Padding.ToUpper(),
+                data_format: conv_utils.convert_data_format(args.DataFormat, 3));
+
+            return outputs;
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/IndexLookup.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Engine;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class IndexLookup : CombinerPreprocessingLayer
+    {
+        public IndexLookup(int max_tokens = -1,
+            int num_oov_indices = 1,
+            string mask_token = "",
+            string oov_token = "[UNK]",
+            string encoding = "utf-8",
+            bool invert = false) : base(new PreprocessingLayerArgs())
+        {
+            var num_mask_tokens = mask_token == null ? 0 : 1;
+            var vocab_size = max_tokens - (num_oov_indices + num_mask_tokens);
+            combiner = new IndexLookupCombiner(vocab_size, mask_token);
+        }
+
+        public override void adapt(IDatasetV2 data, bool reset_state = true)
+        {
+            if (!reset_state)
+                throw new ValueError("IndexLookup does not support streaming adapts.");
+            base.adapt(data, reset_state);
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/IndexLookupAccumulator.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.Engine;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class IndexLookupAccumulator : IAccumulator
+    {
+        public Dictionary<string, int> CountDict { get; set; }
+        public IndexLookupAccumulator()
+        {
+            CountDict = new Dictionary<string, int>();
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/IndexLookupCombiner.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.Engine;
+
+namespace Tensorflow.Keras.Layers
+{
+    /// <summary>
+    /// Combiner for the IndexLookup preprocessing layer.
+    /// </summary>
+    public class IndexLookupCombiner : ICombiner
+    {
+        int _vocab_size;
+        string _mask_value;
+
+        public IndexLookupCombiner(int vocab_size = -1, string mask_value = null)
+        {
+            _vocab_size = vocab_size;
+            _mask_value = mask_value;
+        }
+
+        public void Compute(Tensor values, IAccumulator accumulator = null)
+        {
+            if(accumulator == null)
+            {
+                accumulator = new IndexLookupAccumulator();
+            }
+        }
+
+        public void Deserialize()
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Extract()
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Merge()
+        {
+            throw new NotImplementedException();
+        }
+
+        public IAccumulator Restore()
+        {
+            throw new NotImplementedException();
+        }
+
+        public void Serialize()
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/StringLookup.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Layers
+{
+    /// <summary>
+    /// Maps strings from a vocabulary to integer indices.
+    /// </summary>
+    class StringLookup : IndexLookup
+    {
+        public StringLookup(int max_tokens = -1,
+            int num_oov_indices = 1,
+            string mask_token = "",
+            string[] vocabulary = null,
+            string oov_token = "[UNK]",
+            string encoding = "utf-8",
+            bool invert = false)
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Preprocessing/TextVectorization.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Engine;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class TextVectorization : CombinerPreprocessingLayer
+    {
+        TextVectorizationArgs args;
+        IndexLookup _index_lookup_layer;
+
+        public TextVectorization(TextVectorizationArgs args)
+            : base(args)
+        {
+            this.args = args;
+            args.DType = TF_DataType.TF_STRING;
+            // string standardize = "lower_and_strip_punctuation",
+
+            var mask_token = args.OutputMode == "int" ? "" : null;
+            _index_lookup_layer = new StringLookup(max_tokens: args.MaxTokens,
+                mask_token: mask_token,
+                vocabulary: args.Vocabulary);
+        }
+
+        /// <summary>
+        /// Fits the state of the preprocessing layer to the dataset.
+        /// </summary>
+        /// <param name="data"></param>
+        /// <param name="reset_state"></param>
+        public override void adapt(IDatasetV2 data, bool reset_state = true)
+        {
+            var shape = data.output_shapes[0];
+            if (shape.rank == 1)
+                data = data.map(tensor => array_ops.expand_dims(tensor, -1));
+            build(data.variant_tensor);
+            var preprocessed_inputs = data.map(_preprocess);
+            _index_lookup_layer.adapt(preprocessed_inputs);
+        }
+
+        protected override void build(Tensors inputs)
+        {
+            base.build(inputs);
+        }
+
+        Tensors _preprocess(Tensors inputs)
+        {
+            Tensor input_tensor = null;
+            if (args.Standardize != null)
+                input_tensor = args.Standardize(inputs);
+            if (!string.IsNullOrEmpty(args.Split))
+            {
+                if (inputs.shape.ndim > 1)
+                    input_tensor = array_ops.squeeze(inputs, axis: new[] { -1 });
+                if (args.Split == "whitespace")
+                    input_tensor = tf.strings.split(input_tensor);
+            }
+            return input_tensor;
+        }
+    }
+}

src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using Tensorflow.Framework;
 using Tensorflow.Keras.ArgsDefinition;
 using Tensorflow.Keras.Engine;
@@ -45,7 +46,7 @@ namespace Tensorflow.Keras.Layers
                     return array_ops.reshape(inputs, new[] { batch_dim, -1 });
                 }
 
-                var non_batch_dims = ((int[])input_shape)[1..];
+                var non_batch_dims = ((int[])input_shape).Skip(1).ToArray();
                 var num = 1;
                 if (non_batch_dims.Length > 0)
                 {

src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs

@@ -37,7 +37,7 @@ namespace Tensorflow.Keras.Layers
 
         public override TensorShape ComputeOutputShape(TensorShape input_shape)
         {
-            if (input_shape.dims[1..].Contains(-1))
+            if (input_shape.dims.Skip(1).Contains(-1))
             {
                 throw new NotImplementedException("");
             }

src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.get_training_or_validation_split.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 
 namespace Tensorflow.Keras.Preprocessings
 {
@@ -17,18 +18,21 @@ namespace Tensorflow.Keras.Preprocessings
             float validation_split,
             string subset)
         {
+            if (string.IsNullOrEmpty(subset))
+                return (samples, labels);
+
             var num_val_samples = Convert.ToInt32(samples.Length * validation_split);
             if (subset == "training")
             {
                 Console.WriteLine($"Using {samples.Length - num_val_samples} files for training.");
-                samples = samples[..^num_val_samples];
-                labels = labels[..^num_val_samples];
+                samples = samples.Take(samples.Length - num_val_samples).ToArray();
+                labels = labels.Take(labels.Length - num_val_samples).ToArray();
             }
             else if (subset == "validation")
             {
                 Console.WriteLine($"Using {num_val_samples} files for validation.");
-                samples = samples[(samples.Length - num_val_samples)..];
-                labels = labels[(labels.Length - num_val_samples)..];
+                samples = samples.Skip(samples.Length - num_val_samples).ToArray();
+                labels = labels.Skip(labels.Length - num_val_samples).ToArray();
             }
             else
                 throw new NotImplementedException("");

src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.index_directory.cs

@@ -1,4 +1,5 @@
 using NumSharp;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -21,44 +22,46 @@ namespace Tensorflow.Keras.Preprocessings
         /// file_paths, labels, class_names
         /// </returns>
         public (string[], int[], string[]) index_directory(string directory,
+            string labels,
             string[] formats = null,
             string[] class_names = null,
             bool shuffle = true,
             int? seed = null,
             bool follow_links = false)
         {
-            var labels = new List<int>();
+            var label_list = new List<int>();
             var file_paths = new List<string>();
 
             var class_dirs = Directory.GetDirectories(directory);
-            class_names = class_dirs.Select(x => x.Split(Path.DirectorySeparatorChar)[^1]).ToArray();
+            class_names = class_dirs.Select(x => x.Split(Path.DirectorySeparatorChar).Last()).ToArray();
 
             for (var label = 0; label < class_dirs.Length; label++)
             {
                 var files = Directory.GetFiles(class_dirs[label]);
                 file_paths.AddRange(files);
-                labels.AddRange(Enumerable.Range(0, files.Length).Select(x => label));
+                label_list.AddRange(Enumerable.Range(0, files.Length).Select(x => label));
             }
 
-            var return_labels = labels.Select(x => x).ToArray();
+            var return_labels = label_list.Select(x => x).ToArray();
             var return_file_paths = file_paths.Select(x => x).ToArray();
 
             if (shuffle)
             {
                 if (!seed.HasValue)
                     seed = np.random.randint((long)1e6);
-                var random_index = np.arange(labels.Count);
+                var random_index = np.arange(label_list.Count);
                 var rng = np.random.RandomState(seed.Value);
                 rng.shuffle(random_index);
                 var index = random_index.ToArray<int>();
 
-                for (int i = 0; i < labels.Count; i++)
+                for (int i = 0; i < label_list.Count; i++)
                 {
-                    return_labels[i] = labels[index[i]];
+                    return_labels[i] = label_list[index[i]];
                     return_file_paths[i] = file_paths[index[i]];
                 }
             }
 
+            Console.WriteLine($"Found {return_file_paths.Length} files belonging to {class_names.Length} classes.");
             return (return_file_paths, return_labels, class_names);
         }
     }

src/TensorFlowNET.Keras/Preprocessings/Preprocessing.cs

@@ -1,4 +1,7 @@
-using Tensorflow.Keras.Preprocessings;
+using System;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Layers;
+using Tensorflow.Keras.Preprocessings;
 
 namespace Tensorflow.Keras
 {
@@ -6,5 +9,22 @@ namespace Tensorflow.Keras
     {
         public Sequence sequence => new Sequence();
         public DatasetUtils dataset_utils => new DatasetUtils();
+
+        public TextApi text => _text;
+
+        private static TextApi _text = new TextApi();
+
+        public TextVectorization TextVectorization(Func<Tensor, Tensor> standardize = null,
+            string split = "whitespace",
+            int max_tokens = -1,
+            string output_mode = "int",
+            int output_sequence_length = -1) => new TextVectorization(new TextVectorizationArgs
+            {
+                Standardize = standardize,
+                Split = split,
+                MaxTokens = max_tokens,
+                OutputMode = output_mode,
+                OutputSequenceLength = output_sequence_length
+            });
     }
 }

src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs

@@ -43,6 +43,7 @@ namespace Tensorflow.Keras
                 num_channels = 3;
             
             var (image_paths, label_list, class_name_list) = keras.preprocessing.dataset_utils.index_directory(directory,
+                labels,
                 formats: WHITELIST_FORMATS,
                 class_names: class_names,
                 shuffle: shuffle,
@@ -64,13 +65,30 @@ namespace Tensorflow.Keras
             string[] class_names = null,
             int batch_size = 32,
             bool shuffle = true,
+            int max_length = -1,
             int? seed = null,
             float validation_split = 0.2f,
-            string subset = null)
+            string subset = null,
+            bool follow_links = false)
         {
-           
+            var (file_paths, label_list, class_name_list) = dataset_utils.index_directory(
+               directory,
+               labels,
+               formats: new[] { ".txt" },
+               class_names: class_names,
+               shuffle: shuffle,
+               seed: seed,
+               follow_links: follow_links);
 
-            return null;
+            (file_paths, label_list) = dataset_utils.get_training_or_validation_split(
+                file_paths, label_list, validation_split, subset);
+
+            var dataset = paths_and_labels_to_dataset(file_paths, label_list, label_mode, class_name_list.Length);
+            if (shuffle)
+                dataset = dataset.shuffle(batch_size * 8, seed: seed);
+            dataset = dataset.batch(batch_size);
+            dataset.class_names = class_name_list;
+            return dataset;
         }
     }
 }

src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using static Tensorflow.Binding;
 
 namespace Tensorflow.Keras
@@ -34,5 +35,31 @@ namespace Tensorflow.Keras
             // img.set_shape((image_size[0], image_size[1], num_channels));
             return img;
         }
+
+        public IDatasetV2 paths_and_labels_to_dataset(string[] image_paths,
+            int[] labels,
+            string label_mode,
+            int num_classes,
+            int max_length = -1)
+        {
+            var path_ds = tf.data.Dataset.from_tensor_slices(image_paths);
+            var string_ds = path_ds.map(x => path_to_string_content(x, max_length));
+
+            if (label_mode == "int")
+            {
+                var label_ds = dataset_utils.labels_to_dataset(labels, label_mode, num_classes);
+                string_ds = tf.data.Dataset.zip(string_ds, label_ds);
+            }
+
+            return string_ds;
+        }
+
+        Tensor path_to_string_content(Tensor path, int max_length)
+        {
+            var txt = tf.io.read_file(path);
+            if (max_length > -1)
+                txt = tf.strings.substr(txt, 0, max_length);
+            return txt;
+        }
     }
 }

src/TensorFlowNET.Keras/Preprocessings/Tokenizer.cs

@@ -0,0 +1,444 @@
+using NumSharp;
+using Serilog.Debugging;
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Data.SqlTypes;
+using System.Linq;
+using System.Net.Sockets;
+using System.Text;
+
+namespace Tensorflow.Keras.Text
+{
+    /// <summary>
+    /// Text tokenization API.
+    /// This class allows to vectorize a text corpus, by turning each text into either a sequence of integers 
+    /// (each integer being the index of a token in a dictionary) or into a vector where the coefficient for 
+    /// each token could be binary, based on word count, based on tf-idf...
+    /// </summary>
+    /// <remarks>
+    /// This code is a fairly straight port of the Python code for Keras text preprocessing found at:
+    /// https://github.com/keras-team/keras-preprocessing/blob/master/keras_preprocessing/text.py
+    /// </remarks>
+    public class Tokenizer
+    {
+        private readonly int num_words;
+        private readonly string filters;
+        private readonly bool lower;
+        private readonly char split;
+        private readonly bool char_level;
+        private readonly string oov_token;
+        private readonly Func<string, IEnumerable<string>> analyzer;
+
+        private int document_count = 0;
+
+        private Dictionary<string, int> word_docs = new Dictionary<string, int>();
+        private Dictionary<string, int> word_counts = new Dictionary<string, int>();
+
+        public Dictionary<string, int> word_index = null;
+        public Dictionary<int, string> index_word = null;
+
+        private Dictionary<int, int> index_docs = null;
+
+        public Tokenizer(
+            int num_words = -1,
+            string filters = "!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n",
+            bool lower = true,
+            char split = ' ',
+            bool char_level = false,
+            string oov_token = null,
+            Func<string, IEnumerable<string>> analyzer = null)
+        {
+            this.num_words = num_words;
+            this.filters = filters;
+            this.lower = lower;
+            this.split = split;
+            this.char_level = char_level;
+            this.oov_token = oov_token;
+            this.analyzer = analyzer != null ? analyzer : (text) => TextApi.text_to_word_sequence(text, filters, lower, split);
+        }
+
+        /// <summary>
+        /// Updates internal vocabulary based on a list of texts. 
+        /// </summary>
+        /// <param name="texts">A list of strings, each containing one or more tokens.</param>
+        /// <remarks>Required before using texts_to_sequences or texts_to_matrix.</remarks>
+        public void fit_on_texts(IEnumerable<string> texts)
+        {
+            foreach (var text in texts)
+            {
+                IEnumerable<string> seq = null;
+
+                document_count += 1;
+                if (char_level)
+                {
+                    throw new NotImplementedException("char_level == true");
+                }
+                else
+                {
+                    seq = analyzer(lower ? text.ToLower() : text);
+                }
+
+                foreach (var w in seq)
+                {
+                    var count = 0;
+                    word_counts.TryGetValue(w, out count);
+                    word_counts[w] = count + 1;
+                }
+
+                foreach (var w in new HashSet<string>(seq))
+                {
+                    var count = 0;
+                    word_docs.TryGetValue(w, out count);
+                    word_docs[w] = count + 1;
+                }
+            }
+
+            var wcounts = word_counts.AsEnumerable().ToList();
+            wcounts.Sort((kv1, kv2) => -kv1.Value.CompareTo(kv2.Value));    // Note: '-' gives us descending order.
+
+            var sorted_voc = (oov_token == null) ? new List<string>() : new List<string>() { oov_token };
+            sorted_voc.AddRange(word_counts.Select(kv => kv.Key));
+
+            if (num_words > 0 - 1)
+            {
+                sorted_voc = sorted_voc.Take<string>((oov_token == null) ? num_words : num_words + 1).ToList();
+            }
+
+            word_index = new Dictionary<string, int>(sorted_voc.Count);
+            index_word = new Dictionary<int, string>(sorted_voc.Count);
+            index_docs = new Dictionary<int, int>(word_docs.Count);
+
+            for (int i = 0; i < sorted_voc.Count; i++)
+            {
+                word_index.Add(sorted_voc[i], i + 1);
+                index_word.Add(i + 1, sorted_voc[i]);
+            }
+
+            foreach (var kv in word_docs)
+            {
+                var idx = -1;
+                if (word_index.TryGetValue(kv.Key, out idx))
+                {
+                    index_docs.Add(idx, kv.Value);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Updates internal vocabulary based on a list of texts. 
+        /// </summary>
+        /// <param name="texts">A list of list of strings, each containing one token.</param>
+        /// <remarks>Required before using texts_to_sequences or texts_to_matrix.</remarks>
+        public void fit_on_texts(IEnumerable<IEnumerable<string>> texts)
+        {
+            foreach (var seq in texts)
+            {
+                foreach (var w in seq.Select(s => lower ? s.ToLower() : s))
+                {
+                    var count = 0;
+                    word_counts.TryGetValue(w, out count);
+                    word_counts[w] = count + 1;
+                }
+
+                foreach (var w in new HashSet<string>(word_counts.Keys))
+                {
+                    var count = 0;
+                    word_docs.TryGetValue(w, out count);
+                    word_docs[w] = count + 1;
+                }
+            }
+
+            var wcounts = word_counts.AsEnumerable().ToList();
+            wcounts.Sort((kv1, kv2) => -kv1.Value.CompareTo(kv2.Value));
+
+            var sorted_voc = (oov_token == null) ? new List<string>() : new List<string>() { oov_token };
+            sorted_voc.AddRange(word_counts.Select(kv => kv.Key));
+
+            if (num_words > 0 - 1)
+            {
+                sorted_voc = sorted_voc.Take<string>((oov_token == null) ? num_words : num_words + 1).ToList();
+            }
+
+            word_index = new Dictionary<string, int>(sorted_voc.Count);
+            index_word = new Dictionary<int, string>(sorted_voc.Count);
+            index_docs = new Dictionary<int, int>(word_docs.Count);
+
+            for (int i = 0; i < sorted_voc.Count; i++)
+            {
+                word_index.Add(sorted_voc[i], i + 1);
+                index_word.Add(i + 1, sorted_voc[i]);
+            }
+
+            foreach (var kv in word_docs)
+            {
+                var idx = -1;
+                if (word_index.TryGetValue(kv.Key, out idx))
+                {
+                    index_docs.Add(idx, kv.Value);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Updates internal vocabulary based on a list of sequences.
+        /// </summary>
+        /// <param name="sequences"></param>
+        /// <remarks>Required before using sequences_to_matrix (if fit_on_texts was never called).</remarks>
+        public void fit_on_sequences(IEnumerable<int[]> sequences)
+        {
+            throw new NotImplementedException("fit_on_sequences");
+        }
+
+        /// <summary>
+        /// Transforms each string in texts to a sequence of integers.
+        /// </summary>
+        /// <param name="texts"></param>
+        /// <returns></returns>
+        /// <remarks>Only top num_words-1 most frequent words will be taken into account.Only words known by the tokenizer will be taken into account.</remarks>
+        public IList<int[]> texts_to_sequences(IEnumerable<string> texts)
+        {
+            return texts_to_sequences_generator(texts).ToArray();
+        }
+
+        /// <summary>
+        /// Transforms each token in texts to a sequence of integers.
+        /// </summary>
+        /// <param name="texts"></param>
+        /// <returns></returns>
+        /// <remarks>Only top num_words-1 most frequent words will be taken into account.Only words known by the tokenizer will be taken into account.</remarks>
+        public IList<int[]> texts_to_sequences(IEnumerable<IEnumerable<string>> texts)
+        {
+            return texts_to_sequences_generator(texts).ToArray();
+        }
+
+        public IEnumerable<int[]> texts_to_sequences_generator(IEnumerable<string> texts)
+        {
+            int oov_index = -1;
+            var _ = (oov_token != null) && word_index.TryGetValue(oov_token, out oov_index);
+
+            return texts.Select(text =>
+            {
+                IEnumerable<string> seq = null;
+
+                if (char_level)
+                {
+                    throw new NotImplementedException("char_level == true");
+                }
+                else
+                {
+                    seq = analyzer(lower ? text.ToLower() : text);
+                }
+
+                return ConvertToSequence(oov_index, seq).ToArray();
+            });
+        }
+
+        public IEnumerable<int[]> texts_to_sequences_generator(IEnumerable<IEnumerable<string>> texts)
+        {
+            int oov_index = -1;
+            var _ = (oov_token != null) && word_index.TryGetValue(oov_token, out oov_index);
+            return texts.Select(seq => ConvertToSequence(oov_index, seq).ToArray());
+        }
+
+        private List<int> ConvertToSequence(int oov_index, IEnumerable<string> seq)
+        {
+            var vect = new List<int>();
+            foreach (var w in seq.Select(s => lower ? s.ToLower() : s))
+            {
+                var i = -1;
+                if (word_index.TryGetValue(w, out i))
+                {
+                    if (num_words != -1 && i >= num_words)
+                    {
+                        if (oov_index != -1)
+                        {
+                            vect.Add(oov_index);
+                        }
+                    }
+                    else
+                    {
+                        vect.Add(i);
+                    }
+                }
+                else if (oov_index != -1)
+                {
+                    vect.Add(oov_index);
+                }
+            }
+
+            return vect;
+        }
+
+        /// <summary>
+        /// Transforms each sequence into a list of text.
+        /// </summary>
+        /// <param name="sequences"></param>
+        /// <returns>A list of texts(strings)</returns>
+        /// <remarks>Only top num_words-1 most frequent words will be taken into account.Only words known by the tokenizer will be taken into account.</remarks>
+        public IList<string> sequences_to_texts(IEnumerable<int[]> sequences)
+        {
+            return sequences_to_texts_generator(sequences).ToArray();
+        }
+
+        public IEnumerable<string> sequences_to_texts_generator(IEnumerable<IList<int>> sequences)
+        {
+            int oov_index = -1;
+            var _ = (oov_token != null) && word_index.TryGetValue(oov_token, out oov_index);
+
+            return sequences.Select(seq =>
+            {
+
+                var bldr = new StringBuilder();
+                for (var i = 0; i < seq.Count; i++)
+                {
+                    if (i > 0) bldr.Append(' ');
+
+                    string word = null;
+                    if (index_word.TryGetValue(seq[i], out word))
+                    {
+                        if (num_words != -1 && i >= num_words)
+                        {
+                            if (oov_index != -1)
+                            {
+                                bldr.Append(oov_token);
+                            }
+                        }
+                        else
+                        {
+                            bldr.Append(word);
+                        }
+                    }
+                    else if (oov_index != -1)
+                    {
+                        bldr.Append(oov_token);
+                    }
+                }
+
+                return bldr.ToString();
+            });
+        }
+
+        /// <summary>
+        /// Convert a list of texts to a Numpy matrix.
+        /// </summary>
+        /// <param name="texts">A sequence of strings containing one or more tokens.</param>
+        /// <param name="mode">One of "binary", "count", "tfidf", "freq".</param>
+        /// <returns></returns>
+        public NDArray texts_to_matrix(IEnumerable<string> texts, string mode = "binary")
+        {
+            return sequences_to_matrix(texts_to_sequences(texts), mode);
+        }
+
+        /// <summary>
+        /// Convert a list of texts to a Numpy matrix.
+        /// </summary>
+        /// <param name="texts">A sequence of lists of strings, each containing one token.</param>
+        /// <param name="mode">One of "binary", "count", "tfidf", "freq".</param>
+        /// <returns></returns>
+        public NDArray texts_to_matrix(IEnumerable<IList<string>> texts, string mode = "binary")
+        {
+            return sequences_to_matrix(texts_to_sequences(texts), mode);
+        }
+
+        /// <summary>
+        /// Converts a list of sequences into a Numpy matrix.
+        /// </summary>
+        /// <param name="sequences">A sequence of lists of integers, encoding tokens.</param>
+        /// <param name="mode">One of "binary", "count", "tfidf", "freq".</param>
+        /// <returns></returns>
+        public NDArray sequences_to_matrix(IEnumerable<IList<int>> sequences, string mode = "binary")
+        {
+            if (!modes.Contains(mode)) throw new InvalidArgumentError($"Unknown vectorization mode: {mode}");
+            var word_count = 0;
+
+            if (num_words == -1)
+            {
+                if (word_index != null)
+                {
+                    word_count = word_index.Count + 1;
+                }
+                else
+                {
+                    throw new InvalidOperationException("Specifya dimension ('num_words' arugment), or fit on some text data first.");
+                }
+            }
+            else
+            {
+                word_count = num_words;
+            }
+
+            if (mode == "tfidf" && this.document_count == 0)
+            {
+                throw new InvalidOperationException("Fit the Tokenizer on some text data before using the 'tfidf' mode.");
+            }
+
+            var x = np.zeros(sequences.Count(), word_count);
+
+            for (int i = 0; i < sequences.Count(); i++)
+            {
+                var seq = sequences.ElementAt(i);
+                if (seq == null || seq.Count == 0)
+                    continue;
+
+                var counts = new Dictionary<int, int>();
+
+                var seq_length = seq.Count;
+
+                foreach (var j in seq)
+                {
+                    if (j >= word_count)
+                        continue;
+                    var count = 0;
+                    counts.TryGetValue(j, out count);
+                    counts[j] = count + 1;
+                }
+
+                if (mode == "count")
+                {
+                    foreach (var kv in counts)
+                    {
+                        var j = kv.Key;
+                        var c = kv.Value;
+                        x[i, j] = c;
+                    }
+                }
+                else if (mode == "freq")
+                {
+                    foreach (var kv in counts)
+                    {
+                        var j = kv.Key;
+                        var c = kv.Value;
+                        x[i, j] = ((double)c) / seq_length;
+                    }
+                }
+                else if (mode == "binary")
+                {
+                    foreach (var kv in counts)
+                    {
+                        var j = kv.Key;
+                        var c = kv.Value;
+                        x[i, j] = 1;
+                    }
+                }
+                else if (mode == "tfidf")
+                {
+                    foreach (var kv in counts)
+                    {
+                        var j = kv.Key;
+                        var c = kv.Value;
+                        var id = 0;
+                        var _ = index_docs.TryGetValue(j, out id);
+                        var tf = 1 + np.log(c);
+                        var idf = np.log(1 + document_count / (1 + id));
+                        x[i, j] = tf * idf;
+                    }
+                }
+            }
+
+            return x;
+        }
+
+        private string[] modes = new string[] { "binary", "count", "tfidf", "freq" };
+    }
+}

src/TensorFlowNET.Keras/Sequence.cs

@@ -15,7 +15,9 @@
 ******************************************************************************/
 
 using NumSharp;
+using NumSharp.Utilities;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace Tensorflow.Keras
@@ -34,14 +36,18 @@ namespace Tensorflow.Keras
         /// <param name="truncating">String, 'pre' or 'post'</param>
         /// <param name="value">Float or String, padding value.</param>
         /// <returns></returns>
-        public NDArray pad_sequences(NDArray sequences,
+        public NDArray pad_sequences(IEnumerable<int[]> sequences,
             int? maxlen = null,
             string dtype = "int32",
             string padding = "pre",
             string truncating = "pre",
             object value = null)
         {
-            int[] length = new int[sequences.size];
+            if (value != null) throw new NotImplementedException("padding with a specific value.");
+            if (padding != "pre" && padding != "post") throw new InvalidArgumentError("padding must be 'pre' or 'post'.");
+            if (truncating != "pre" && truncating != "post") throw new InvalidArgumentError("truncating must be 'pre' or 'post'.");
+
+            var length = sequences.Select(s => s.Length);
 
             if (maxlen == null)
                 maxlen = length.Max();
@@ -49,19 +55,26 @@ namespace Tensorflow.Keras
             if (value == null)
                 value = 0f;
 
-            var nd = new NDArray(np.int32, new Shape(sequences.size, maxlen.Value));
-#pragma warning disable CS0162 // Unreachable code detected
+            var type = getNPType(dtype);
+            var nd = new NDArray(type, new Shape(length.Count(), maxlen.Value), true);
+
             for (int i = 0; i < nd.shape[0]; i++)
-#pragma warning restore CS0162 // Unreachable code detected
             {
-                switch (sequences[i])
+                var s = sequences.ElementAt(i);
+                if (s.Length > maxlen.Value)
                 {
-                    default:
-                        throw new NotImplementedException("pad_sequences");
+                    s = (truncating == "pre") ? s.Slice(s.Length - maxlen.Value, s.Length) : s.Slice(0, maxlen.Value);
                 }
+                var sliceString = (padding == "pre") ? $"{i},{maxlen - s.Length}:" : $"{i},:{s.Length}";
+                nd[sliceString] = np.array(s);
             }
 
             return nd;
         }
+
+        private Type getNPType(string typeName)
+        {
+            return System.Type.GetType("NumSharp.np,NumSharp").GetField(typeName).GetValue(null) as Type;
+        }
     }
 }

src/TensorFlowNET.Keras/Tensorflow.Keras.csproj

@@ -6,7 +6,7 @@
     <LangVersion>8.0</LangVersion>
     <RootNamespace>Tensorflow.Keras</RootNamespace>
     <Platforms>AnyCPU;x64</Platforms>
-    <Version>0.4.0</Version>
+    <Version>0.5.0</Version>
     <Authors>Haiping Chen</Authors>
     <Product>Keras for .NET</Product>
     <Copyright>Apache 2.0, Haiping Chen 2020</Copyright>
@@ -23,7 +23,8 @@
 * Implemented backward_function.
 * Support model.load_weights.
 * Add Subtract layer
-* Support YOLOv3 model.</PackageReleaseNotes>
+* Support YOLOv3 model.
+* Text preprocessing</PackageReleaseNotes>
     <Description>Keras for .NET
 
 Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent &amp; simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear &amp; actionable error messages.</Description>
@@ -34,8 +35,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
     <RepositoryType>Git</RepositoryType>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>Open.snk</AssemblyOriginatorKeyFile>
-    <AssemblyVersion>0.4.0.0</AssemblyVersion>
-    <FileVersion>0.4.0.0</FileVersion>
+    <AssemblyVersion>0.5.0.0</AssemblyVersion>
+    <FileVersion>0.5.0.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
 
@@ -48,6 +49,10 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
     <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <DocumentationFile>Tensorflow.Keras.xml</DocumentationFile>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.138" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
@@ -62,10 +67,6 @@ Keras is an API designed for human beings, not machines. Keras follows best prac
     </None>
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Engine\Interfaces\" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
   </ItemGroup>

src/TensorFlowNET.Keras/TextApi.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Tensorflow.Keras.Text;
+
+namespace Tensorflow.Keras
+{
+    public class TextApi
+    {
+        public Tensorflow.Keras.Text.Tokenizer Tokenizer(
+                int num_words = -1,
+                string filters = DefaultFilter,
+                bool lower = true,
+                char split = ' ',
+                bool char_level = false,
+                string oov_token = null,
+                Func<string, IEnumerable<string>> analyzer = null)
+        {
+            return new Keras.Text.Tokenizer(num_words, filters, lower, split, char_level, oov_token, analyzer);
+        }
+
+        public static IEnumerable<string> text_to_word_sequence(string text, string filters = DefaultFilter, bool lower = true, char split = ' ')
+        {
+            if (lower)
+            {
+                text = text.ToLower();
+            }
+            var newText = new String(text.Where(c => !filters.Contains(c)).ToArray());
+            return newText.Split(split);
+        }
+
+        private const string DefaultFilter = "!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n";
+    }
+}

src/TensorFlowNET.Keras/Utils/KerasUtils.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Utils
+{
+    public class KerasUtils
+    {
+        /// <summary>
+        /// Downloads a file from a URL if it not already in the cache.
+        /// </summary>
+        /// <param name="fname">Name of the file</param>
+        /// <param name="origin">Original URL of the file</param>
+        /// <param name="untar"></param>
+        /// <param name="md5_hash"></param>
+        /// <param name="file_hash"></param>
+        /// <param name="cache_subdir"></param>
+        /// <param name="hash_algorithm"></param>
+        /// <param name="extract"></param>
+        /// <param name="archive_format"></param>
+        /// <param name="cache_dir"></param>
+        /// <returns></returns>
+        public string get_file(string fname, string origin,
+            bool untar = false,
+            string md5_hash = null,
+            string file_hash = null,
+            string cache_subdir = "datasets",
+            string hash_algorithm = "auto",
+            bool extract = false,
+            string archive_format = "auto",
+            string cache_dir = null)
+            => data_utils.get_file(fname, origin, 
+                untar: untar,
+                md5_hash: md5_hash,
+                file_hash: file_hash,
+                cache_subdir: cache_subdir,
+                hash_algorithm: hash_algorithm,
+                extract: extract,
+                archive_format: archive_format,
+                cache_dir: cache_dir);
+    }
+}

src/TensorFlowNET.Keras/Utils/RuntimeHelpers.cs

@@ -1,39 +0,0 @@
-namespace System.Runtime.CompilerServices
-{
-    internal static class RuntimeHelpers
-    {
-        /// <summary>
-        /// Slices the specified array using the specified range.
-        /// </summary>
-        public static T[] GetSubArray<T>(T[] array, Range range)
-        {
-            if (array == null)
-            {
-                throw new ArgumentNullException(nameof(array));
-            }
-
-            (int offset, int length) = range.GetOffsetAndLength(array.Length);
-
-            if (default(T) != null || typeof(T[]) == array.GetType())
-            {
-                // We know the type of the array to be exactly T[].
-
-                if (length == 0)
-                {
-                    return Array.Empty<T>();
-                }
-
-                var dest = new T[length];
-                Array.Copy(array, offset, dest, 0, length);
-                return dest;
-            }
-            else
-            {
-                // The array is actually a U[] where U:T.
-                var dest = (T[])Array.CreateInstance(array.GetType().GetElementType(), length);
-                Array.Copy(array, offset, dest, 0, length);
-                return dest;
-            }
-        }
-    }
-}

src/TensorFlowNET.Keras/Utils/data_utils.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Tensorflow.Keras.Utils
+{
+    public class data_utils
+    {
+        public static string get_file(string fname, string origin,
+            bool untar = false,
+            string md5_hash = null,
+            string file_hash = null,
+            string cache_subdir = "datasets",
+            string hash_algorithm = "auto",
+            bool extract = false,
+            string archive_format = "auto",
+            string cache_dir = null)
+        {
+            var datadir_base = cache_dir;
+            Directory.CreateDirectory(datadir_base);
+
+            var datadir = Path.Combine(datadir_base, cache_subdir);
+            Directory.CreateDirectory(datadir);
+
+            Web.Download(origin, datadir, fname);
+
+            if (untar)
+                Compress.ExtractTGZ(Path.Combine(datadir_base, fname), datadir_base);
+            else if (extract)
+                Compress.ExtractGZip(Path.Combine(datadir_base, fname), datadir_base);
+
+            return datadir;
+        }
+    }
+}

src/TensorFlowNET.Keras/Utils/layer_utils.cs

@@ -67,7 +67,7 @@ namespace Tensorflow.Keras.Utils
                     line_length = 65;
                 if (positions == null)
                     positions = new[] { 0.45f, 0.85f, 1.0f };
-                if (positions[^1] <= 1)
+                if (positions.Last() <= 1)
                     positions = positions.Select(p => line_length * p).ToArray();
                 to_display = new[] { "Layer (type)", "Output Shape", "Param #" };
             }
@@ -77,7 +77,7 @@ namespace Tensorflow.Keras.Utils
                     line_length = 98;
                 if (positions == null)
                     positions = new[] { 0.33f, 0.55f, 0.67f, 1.0f };
-                if (positions[^1] <= 1)
+                if (positions.Last() <= 1)
                     positions = positions.Select(p => line_length * p).ToArray();
                 to_display = new[] { "Layer (type)", "Output Shape", "Param #", "Connected to" };
 
@@ -118,7 +118,7 @@ namespace Tensorflow.Keras.Utils
             foreach (var i in range(fields.Length))
             {
                 if (i > 0)
-                    line = line[0..^1] + " ";
+                    line = line + " ";
                 line += fields[i];
                 line = string.Join("", line.Take(positions[i]));
                 line += string.Join("", range(positions[i] - len(line)).Select(x => " "));

src/TensorFlowNET.Text/Tokenizers/WhitespaceTokenizer.cs

@@ -1,6 +1,8 @@
-using System;
+using NumSharp;
+using System;
 using System.Collections.Generic;
 using System.Text;
+using static Tensorflow.Binding;
 
 namespace Tensorflow.Text.Tokenizers
 {
@@ -13,7 +15,31 @@ namespace Tensorflow.Text.Tokenizers
         /// <returns></returns>
         public Tensor tokenize(Tensor input)
         {
+            tokenize_with_offsets(input);
             throw new NotImplementedException("");
         }
+
+        Tensor[] tokenize_with_offsets(Tensor input)
+        {
+            tf_with(ops.name_scope(null, "WhitespaceTokenize"), scope =>
+            {
+                _whitespace_tokenize_with_offsets_encode_decode_wrapper(input);
+            });
+            throw new NotImplementedException("");
+        }
+
+        Tensor _whitespace_tokenize_with_offsets_encode_decode_wrapper(Tensor input_tensor)
+        {
+            // Decode the strings and get byte offsets
+            var (codepoints, byte_start_offsets) = tf.strings.unicode_decode_with_offsets(input_tensor, "UTF-8");
+            var byte_end_offsets = array_ops.concat(new Tensor[] 
+            {
+                byte_start_offsets[Slice.All, new Slice(1)],
+                math_ops.cast(
+                    array_ops.expand_dims(tf.strings.string_length(input_tensor), 1),
+                    dtypes.int64)
+            }, 1);
+            return input_tensor;
+        }
     }
 }

src/TensorFlowNet.Benchmarks/Tensorflow.Benchmark.csproj

@@ -2,13 +2,14 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <Platforms>AnyCPU;x64</Platforms>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <PlatformTarget>x64</PlatformTarget>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

tensorflowlib/README.md

@@ -56,7 +56,7 @@ Set ENV `BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\
 
 1. Build static library
 
-`bazel build --output_base=C:/tmp/tfcompilation build --config=opt //tensorflow:tensorflow`
+`bazel build --output_base=C:/tmp/tfcompilation --config=opt //tensorflow:tensorflow`
 
 2. Build pip package
 
@@ -70,6 +70,16 @@ Set ENV `BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\
 
 `pip install C:/tmp/tensorflow_pkg/tensorflow-1.15.0-cp36-cp36m-win_amd64.whl`
 
+### Build from source for MacOS
+
+```shell
+$ cd /usr/local/lib/bazel/bin
+$ curl -LO https://release.bazel.build/3.7.2/release/bazel-3.7.2-darwin-x86_64
+$ chmod +x bazel-3.7.2-darwin-x86_64
+$ cd ~/Projects/tensorflow
+$ bazel build --config=opt //tensorflow:tensorflow
+```
+
 ### Build specific version for tf.net
 
 https://github.com/SciSharp/tensorflow

test/TensorFlowNET.Keras.UnitTest/Layers/PoolingTest.cs

@@ -0,0 +1,305 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using NumSharp;
+using System.Linq;
+using Tensorflow;
+using static Tensorflow.Binding;
+using static Tensorflow.KerasApi;
+
+namespace TensorFlowNET.Keras.UnitTest
+{
+    /// <summary>
+    /// https://www.tensorflow.org/versions/r2.3/api_docs/python/tf/keras/layers
+    /// </summary>
+    [TestClass]
+    public class PoolingTest : EagerModeTestBase
+    {
+        private NDArray input_array_1D = np.array(new float[,,]
+            {
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+            });
+
+        private NDArray input_array_2D = np.array(new float[,,,]
+            {{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+                },{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+            }});
+
+        [TestMethod]
+        public void GlobalAverage1DPoolingChannelsLast()
+        {
+            var pool = keras.layers.GlobalAveragePooling1D();
+            var y = pool.Apply(input_array_1D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(5, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {1,2,3,3,3},
+                {4,5,6,3,3},
+                {7,8,9,3,3},
+                {7,8,9,3,3}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalAverage1DPoolingChannelsFirst()
+        {
+            var pool = keras.layers.GlobalAveragePooling1D(data_format: "channels_first");
+            var y = pool.Apply(input_array_1D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(3, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {2.4f, 2.4f, 2.4f},
+                {4.2f, 4.2f, 4.2f},
+                {6.0f, 6.0f, 6.0f},
+                {6.0f, 6.0f, 6.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalAverage2DPoolingChannelsLast()
+        {
+            var pool = keras.layers.GlobalAveragePooling2D();
+            var y = pool.Apply(input_array_2D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(5, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {2.5f, 3.5f, 4.5f, 3.0f, 3.0f},
+                {7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                {2.5f, 3.5f, 4.5f, 3.0f, 3.0f},
+                {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalAverage2DPoolingChannelsFirst()
+        {
+            var pool = keras.layers.GlobalAveragePooling2D(data_format: "channels_first");
+            var y = pool.Apply(input_array_2D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(2, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {2.4f, 4.2f},
+                {6.0f, 6.0f},
+                {2.4f, 4.2f},
+                {6.0f, 6.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalMax1DPoolingChannelsLast()
+        {
+            var pool = keras.layers.GlobalMaxPooling1D();
+            var y = pool.Apply(input_array_1D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(5, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {1,2,3,3,3},
+                {4,5,6,3,3},
+                {7,8,9,3,3},
+                {7,8,9,3,3}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalMax1DPoolingChannelsFirst()
+        {
+            var pool = keras.layers.GlobalMaxPooling1D(data_format: "channels_first");
+            var y = pool.Apply(input_array_1D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(3, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {3.0f, 3.0f, 3.0f},
+                {6.0f, 6.0f, 6.0f},
+                {9.0f, 9.0f, 9.0f},
+                {9.0f, 9.0f, 9.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalMax2DPoolingChannelsLast()
+        {
+            var input_array_2D = np.array(new float[,,,]
+            {{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,9,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+                },{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,9}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+            }});
+
+            var pool = keras.layers.GlobalMaxPooling2D();
+            var y = pool.Apply(input_array_2D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(5, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {4.0f, 5.0f, 6.0f, 9.0f, 3.0f},
+                {7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                {4.0f, 5.0f, 6.0f, 3.0f, 9.0f},
+                {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void GlobalMax2DPoolingChannelsFirst()
+        {
+            var input_array_2D = np.array(new float[,,,]
+            {{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,9,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+                },{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,9}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+            }});
+
+            var pool = keras.layers.GlobalMaxPooling2D(data_format: "channels_first");
+            var y = pool.Apply(input_array_2D);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(2, y.shape[1]);
+
+            var expected = np.array(new float[,]
+            {
+                {9.0f, 6.0f},
+                {9.0f, 9.0f},
+                {9.0f, 6.0f},
+                {9.0f, 9.0f}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod, Ignore("There's an error generated from TF complaining about the shape of the pool. Needs further investigation.")]
+        public void Max1DPoolingChannelsLast()
+        {
+            var x = input_array_1D;
+            var pool = keras.layers.MaxPooling1D(pool_size:2, strides:1);
+            var y = pool.Apply(x);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(2, y.shape[1]);
+            Assert.AreEqual(5, y.shape[2]);
+
+            var expected = np.array(new float[,,]
+            {
+                {{2.0f, 2.0f, 3.0f, 3.0f, 3.0f},
+                 { 1.0f, 2.0f, 3.0f, 3.0f, 3.0f}},
+
+                {{4.0f, 5.0f, 6.0f, 3.0f, 3.0f},
+                 {4.0f, 5.0f, 6.0f, 3.0f, 3.0f}},
+
+                {{7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                 {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}},
+
+                {{7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                 {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+
+        [TestMethod]
+        public void Max2DPoolingChannelsLast()
+        {
+            var x = np.array(new float[,,,]
+            {{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,9,3}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+                },{
+                {{1,2,3,3,3},{1,2,3,3,3},{1,2,3,3,9}},
+                {{4,5,6,3,3},{4,5,6,3,3},{4,5,6,3,3}},
+                },{
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}},
+                {{7,8,9,3,3},{7,8,9,3,3},{7,8,9,3,3}}
+            }});
+
+            var pool = keras.layers.MaxPooling2D(pool_size: 2, strides: 1);
+            var y = pool.Apply(x);
+
+            Assert.AreEqual(4, y.shape[0]);
+            Assert.AreEqual(1, y.shape[1]);
+            Assert.AreEqual(2, y.shape[2]);
+            Assert.AreEqual(5, y.shape[3]);
+
+            var expected = np.array(new float[,,,]
+            {
+                {{{4.0f, 5.0f, 6.0f, 3.0f, 3.0f},
+                 {4.0f, 5.0f, 6.0f, 9.0f, 3.0f}}},
+
+
+               {{{7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                 {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}}},
+
+
+               {{{4.0f, 5.0f, 6.0f, 3.0f, 3.0f},
+                 {4.0f, 5.0f, 6.0f, 3.0f, 9.0f}}},
+
+
+               {{{7.0f, 8.0f, 9.0f, 3.0f, 3.0f},
+                 {7.0f, 8.0f, 9.0f, 3.0f, 3.0f}}}
+            });
+
+            Assert.AreEqual(expected, y[0].numpy());
+        }
+    }
+}

test/TensorFlowNET.Keras.UnitTest/PreprocessingTests.cs

@@ -0,0 +1,413 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Text;
+using NumSharp;
+using static Tensorflow.KerasApi;
+using Tensorflow;
+using Tensorflow.Keras.Datasets;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace TensorFlowNET.Keras.UnitTest
+{
+    [TestClass]
+    public class PreprocessingTests : EagerModeTestBase
+    {
+        private readonly string[] texts = new string[] {
+                "It was the best of times, it was the worst of times.",
+                "Mr and Mrs Dursley of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much.",
+                "It was the best of times, it was the worst of times.",
+                "Mr and Mrs Dursley of number four, Privet Drive.",
+                };
+
+        private readonly string[][] tokenized_texts = new string[][] {
+                new string[] {"It","was","the","best","of","times","it","was","the","worst","of","times"},
+                new string[] {"mr","and","mrs","dursley","of","number","four","privet","drive","were","proud","to","say","that","they","were","perfectly","normal","thank","you","very","much"},
+                new string[] {"It","was","the","best","of","times","it","was","the","worst","of","times"},
+                new string[] {"mr","and","mrs","dursley","of","number","four","privet","drive"},
+                };
+
+        private readonly string[] processed_texts = new string[] {
+                "it was the best of times it was the worst of times",
+                "mr and mrs dursley of number four privet drive were proud to say that they were perfectly normal thank you very much",
+                "it was the best of times it was the worst of times",
+                "mr and mrs dursley of number four privet drive",
+                };
+
+        private const string OOV = "<OOV>";
+
+        [TestMethod]
+        public void TokenizeWithNoOOV()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            Assert.AreEqual(7, tokenizer.word_index["worst"]);
+            Assert.AreEqual(12, tokenizer.word_index["number"]);
+            Assert.AreEqual(16, tokenizer.word_index["were"]);
+        }
+
+        [TestMethod]
+        public void TokenizeWithNoOOV_Tkn()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            // Use the list version, where the tokenization has already been done.
+            tokenizer.fit_on_texts(tokenized_texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            Assert.AreEqual(7, tokenizer.word_index["worst"]);
+            Assert.AreEqual(12, tokenizer.word_index["number"]);
+            Assert.AreEqual(16, tokenizer.word_index["were"]);
+        }
+
+        [TestMethod]
+        public void TokenizeWithOOV()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(28, tokenizer.word_index.Count);
+
+            Assert.AreEqual(1,  tokenizer.word_index[OOV]);
+            Assert.AreEqual(8,  tokenizer.word_index["worst"]);
+            Assert.AreEqual(13, tokenizer.word_index["number"]);
+            Assert.AreEqual(17, tokenizer.word_index["were"]);
+        }
+
+        [TestMethod]
+        public void TokenizeWithOOV_Tkn()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            // Use the list version, where the tokenization has already been done.
+            tokenizer.fit_on_texts(tokenized_texts);
+
+            Assert.AreEqual(28, tokenizer.word_index.Count);
+
+            Assert.AreEqual(1, tokenizer.word_index[OOV]);
+            Assert.AreEqual(8, tokenizer.word_index["worst"]);
+            Assert.AreEqual(13, tokenizer.word_index["number"]);
+            Assert.AreEqual(17, tokenizer.word_index["were"]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequences()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], sequences[0][9]);
+            Assert.AreEqual(tokenizer.word_index["proud"], sequences[1][10]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequences_Tkn()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            // Use the list version, where the tokenization has already been done.
+            tokenizer.fit_on_texts(tokenized_texts);
+
+            var sequences = tokenizer.texts_to_sequences(tokenized_texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], sequences[0][9]);
+            Assert.AreEqual(tokenizer.word_index["proud"], sequences[1][10]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequencesAndBack()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            var processed = tokenizer.sequences_to_texts(sequences);
+
+            Assert.AreEqual(4, processed.Count);
+
+            for (var i = 0; i < processed.Count; i++)
+                Assert.AreEqual(processed_texts[i], processed[i]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequencesAndBack_Tkn1()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            // Use the list version, where the tokenization has already been done.
+            tokenizer.fit_on_texts(tokenized_texts);
+
+            // Use the list version, where the tokenization has already been done.
+            var sequences = tokenizer.texts_to_sequences(tokenized_texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            var processed = tokenizer.sequences_to_texts(sequences);
+
+            Assert.AreEqual(4, processed.Count);
+
+            for (var i = 0; i < processed.Count; i++)
+                Assert.AreEqual(processed_texts[i], processed[i]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequencesAndBack_Tkn2()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            // Use the list version, where the tokenization has already been done.
+            tokenizer.fit_on_texts(tokenized_texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            var processed = tokenizer.sequences_to_texts(sequences);
+
+            Assert.AreEqual(4, processed.Count);
+
+            for (var i = 0; i < processed.Count; i++)
+                Assert.AreEqual(processed_texts[i], processed[i]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequencesAndBack_Tkn3()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            // Use the list version, where the tokenization has already been done.
+            var sequences = tokenizer.texts_to_sequences(tokenized_texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            var processed = tokenizer.sequences_to_texts(sequences);
+
+            Assert.AreEqual(4, processed.Count);
+
+            for (var i = 0; i < processed.Count; i++)
+                Assert.AreEqual(processed_texts[i], processed[i]);
+        }
+        [TestMethod]
+        public void TokenizeTextsToSequencesWithOOV()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], sequences[0][9]);
+            Assert.AreEqual(tokenizer.word_index["proud"], sequences[1][10]);
+
+            for (var i = 0; i < sequences.Count; i++)
+                for (var j = 0; j < sequences[i].Length; j++)
+                Assert.AreNotEqual(tokenizer.word_index[OOV], sequences[i][j]);
+        }
+
+        [TestMethod]
+        public void TokenizeTextsToSequencesWithOOVPresent()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV, num_words:20);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            Assert.AreEqual(4, sequences.Count);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], sequences[0][9]);
+            Assert.AreEqual(tokenizer.word_index["proud"], sequences[1][10]);
+
+            var oov_count = 0;
+            for (var i = 0; i < sequences.Count; i++)
+                for (var j = 0; j < sequences[i].Length; j++)
+                    if (tokenizer.word_index[OOV] == sequences[i][j])
+                        oov_count += 1;
+
+            Assert.AreEqual(9, oov_count);
+        }
+
+        [TestMethod]
+        public void PadSequencesWithDefaults()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            var padded = keras.preprocessing.sequence.pad_sequences(sequences);
+
+            Assert.AreEqual(4, padded.shape[0]);
+            Assert.AreEqual(22, padded.shape[1]);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], padded[0, 19].GetInt32());
+            for (var i = 0; i < 8; i++)
+                Assert.AreEqual(0, padded[0, i].GetInt32());
+            Assert.AreEqual(tokenizer.word_index["proud"], padded[1, 10].GetInt32());
+            for (var i = 0; i < 20; i++)
+                Assert.AreNotEqual(0, padded[1, i].GetInt32());
+        }
+
+        [TestMethod]
+        public void PadSequencesPrePaddingTrunc()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            var padded = keras.preprocessing.sequence.pad_sequences(sequences,maxlen:15);
+
+            Assert.AreEqual(4, padded.shape[0]);
+            Assert.AreEqual(15, padded.shape[1]);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], padded[0, 12].GetInt32());
+            for (var i = 0; i < 3; i++)
+                Assert.AreEqual(0, padded[0, i].GetInt32());
+            Assert.AreEqual(tokenizer.word_index["proud"], padded[1, 3].GetInt32());
+            for (var i = 0; i < 15; i++)
+                Assert.AreNotEqual(0, padded[1, i].GetInt32());
+        }
+
+        [TestMethod]
+        public void PadSequencesPrePaddingTrunc_Larger()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            var padded = keras.preprocessing.sequence.pad_sequences(sequences, maxlen: 45);
+
+            Assert.AreEqual(4, padded.shape[0]);
+            Assert.AreEqual(45, padded.shape[1]);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], padded[0, 42].GetInt32());
+            for (var i = 0; i < 33; i++)
+                Assert.AreEqual(0, padded[0, i].GetInt32());
+            Assert.AreEqual(tokenizer.word_index["proud"], padded[1, 33].GetInt32());
+        }
+
+        [TestMethod]
+        public void PadSequencesPostPaddingTrunc()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            var padded = keras.preprocessing.sequence.pad_sequences(sequences, maxlen: 15, padding: "post", truncating: "post");
+
+            Assert.AreEqual(4, padded.shape[0]);
+            Assert.AreEqual(15, padded.shape[1]);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], padded[0, 9].GetInt32());
+            for (var i = 12; i < 15; i++)
+                Assert.AreEqual(0, padded[0, i].GetInt32());
+            Assert.AreEqual(tokenizer.word_index["proud"], padded[1, 10].GetInt32());
+            for (var i = 0; i < 15; i++)
+                Assert.AreNotEqual(0, padded[1, i].GetInt32());
+        }
+
+        [TestMethod]
+        public void PadSequencesPostPaddingTrunc_Larger()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer(oov_token: OOV);
+            tokenizer.fit_on_texts(texts);
+
+            var sequences = tokenizer.texts_to_sequences(texts);
+            var padded = keras.preprocessing.sequence.pad_sequences(sequences, maxlen: 45, padding: "post", truncating: "post");
+
+            Assert.AreEqual(4, padded.shape[0]);
+            Assert.AreEqual(45, padded.shape[1]);
+
+            Assert.AreEqual(tokenizer.word_index["worst"], padded[0, 9].GetInt32());
+            for (var i = 32; i < 45; i++)
+                Assert.AreEqual(0, padded[0, i].GetInt32());
+            Assert.AreEqual(tokenizer.word_index["proud"], padded[1, 10].GetInt32());
+        }
+
+        [TestMethod]
+        public void TextToMatrixBinary()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            var matrix = tokenizer.texts_to_matrix(texts);
+
+            Assert.AreEqual(texts.Length, matrix.shape[0]);
+
+            CompareLists(new double[] { 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, matrix[0].ToArray<double>());
+            CompareLists(new double[] { 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, matrix[1].ToArray<double>());
+        }
+
+        [TestMethod]
+        public void TextToMatrixCount()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            var matrix = tokenizer.texts_to_matrix(texts, mode:"count");
+
+            Assert.AreEqual(texts.Length, matrix.shape[0]);
+
+            CompareLists(new double[] { 0, 2, 2, 2, 1, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, matrix[0].ToArray<double>());
+            CompareLists(new double[] { 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, matrix[1].ToArray<double>());
+        }
+
+        [TestMethod]
+        public void TextToMatrixFrequency()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            var matrix = tokenizer.texts_to_matrix(texts, mode: "freq");
+
+            Assert.AreEqual(texts.Length, matrix.shape[0]);
+
+            double t12 = 2.0 / 12.0;
+            double o12 = 1.0 / 12.0;
+            double t22 = 2.0 / 22.0;
+            double o22 = 1.0 / 22.0;
+
+            CompareLists(new double[] { 0, t12, t12, t12, o12, t12, t12, o12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, matrix[0].ToArray<double>());
+            CompareLists(new double[] { 0, 0, 0, 0, 0, o22, 0, 0, o22, o22, o22, o22, o22, o22, o22, o22, t22, o22, o22, o22, o22, o22, o22, o22, o22, o22, o22, o22 }, matrix[1].ToArray<double>());
+        }
+
+        [TestMethod]
+        public void TextToMatrixTDIDF()
+        {
+            var tokenizer = keras.preprocessing.text.Tokenizer();
+            tokenizer.fit_on_texts(texts);
+
+            Assert.AreEqual(27, tokenizer.word_index.Count);
+
+            var matrix = tokenizer.texts_to_matrix(texts, mode: "tfidf");
+
+            Assert.AreEqual(texts.Length, matrix.shape[0]);
+
+            double t1 = 1.1736001944781467;
+            double t2 = 0.69314718055994529;
+            double t3 = 1.860112299086919;
+            double t4 = 1.0986122886681098;
+            double t5 = 0.69314718055994529;
+
+            CompareLists(new double[] { 0, t1, t1, t1, t2, 0, t1, t2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, matrix[0].ToArray<double>());
+            CompareLists(new double[] { 0, 0, 0, 0, 0, 0, 0, 0, t5, t5, t5, t5, t5, t5, t5, t5, t3, t4, t4, t4, t4, t4, t4, t4, t4, t4, t4, t4 }, matrix[1].ToArray<double>());
+        }
+
+        private void CompareLists<T>(IList<T> expected, IList<T> actual)
+        {
+            Assert.AreEqual(expected.Count, actual.Count);
+            for (var i = 0; i < expected.Count; i++)
+            {
+                Assert.AreEqual(expected[i], actual[i]);
+            }
+        }
+
+    }
+}