impplementing NB classifier

.gitignore

@@ -328,15 +328,8 @@ ASALocalRun/
 
 # MFractors (Xamarin productivity tool) working folder 
 .mfractor/
-/tensorflowlib/win7-x64/native/libtensorflow.dll
-/tensorflowlib/osx/native/libtensorflow_framework.dylib
-/tensorflowlib/osx/native/libtensorflow.dylib
-/tensorflowlib/linux/native/libtensorflow_framework.so
-/tensorflowlib/linux/native/libtensorflow.so
-/src/TensorFlowNET.Core/tensorflow.dll
 /docs/build
-src/TensorFlowNET.Native/libtensorflow.dll
 src/TensorFlowNET.Native/bazel-*
-/src/TensorFlowNET.Native/libtensorflow.lib
 src/TensorFlowNET.Native/c_api.h
 /.vscode
+test/TensorFlowNET.Examples/mnist

README.md

@@ -72,6 +72,8 @@ Read the docs & book [The Definitive Guide to Tensorflow.NET](https://tensorflow
 * [Basic Operations](test/TensorFlowNET.Examples/BasicOperations.cs)
 * [Image Recognition](test/TensorFlowNET.Examples/ImageRecognition.cs)
 * [Linear Regression](test/TensorFlowNET.Examples/LinearRegression.cs)
+* [Logistic Regression](test/TensorFlowNET.Examples/LogisticRegression.cs)
+* [Nearest Neighbor](test/TensorFlowNET.Examples/NearestNeighbor.cs)
 * [Text Classification](test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs)
 * [CNN Text Classification](test/TensorFlowNET.Examples/CnnTextClassification.cs)
 * [Naive Bayes Classification](test/TensorFlowNET.Examples/NaiveBayesClassifier.cs)

TensorFlow.NET.sln

@@ -11,8 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Core", "src\T
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Visualization", "src\TensorFlowNET.Visualization\TensorFlowNET.Visualization.csproj", "{0254BFF9-453C-4FE0-9609-3644559A79CE}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NumSharp.Core", "..\NumSharp\src\NumSharp.Core\NumSharp.Core.csproj", "{3EEAFB06-BEF0-4261-BAAB-630EABD25290}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -35,10 +33,6 @@ Global
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

docs/source/Graph.md

@@ -21,3 +21,61 @@ A typical graph is looks like below:
 
 ![image](../assets/graph_vis_animation.gif)
 
+
+
+### Save Model
+
+Saving the model means saving all the values of the parameters and the graph.
+
+```python
+saver = tf.train.Saver()
+saver.save(sess,'./tensorflowModel.ckpt')
+```
+
+After saving the model there will be four files:
+
+* tensorflowModel.ckpt.meta:
+* tensorflowModel.ckpt.data-00000-of-00001:
+* tensorflowModel.ckpt.index
+* checkpoint
+
+We also created a protocol buffer file .pbtxt. It is human readable if you want to convert it to binary: `as_text: false`.
+
+* tensorflowModel.pbtxt: 
+
+This holds a network of nodes, each representing one operation, connected to each other as inputs and outputs.
+
+
+
+### Freezing the Graph
+
+##### *Why we need it?*
+
+When we need to keep all the values of the variables and the Graph structure in a single file we have to freeze the graph.
+
+```csharp
+from tensorflow.python.tools import freeze_graph
+
+freeze_graph.freeze_graph(input_graph = 'logistic_regression/tensorflowModel.pbtxt', 
+                              input_saver = "", 
+                              input_binary = False, 
+                              input_checkpoint = 'logistic_regression/tensorflowModel.ckpt', 
+                              output_node_names = "Softmax",
+                              restore_op_name = "save/restore_all", 
+                              filename_tensor_name = "save/Const:0",
+                              output_graph = 'frozentensorflowModel.pb', 
+                              clear_devices = True, 
+                              initializer_nodes = "")
+
+```
+
+### Optimizing for Inference
+
+To Reduce the amount of computation needed when the network is used only for inferences we can remove some parts of a graph that are only needed for training. 
+
+
+
+### Restoring the Model
+
+
+

docs/source/LinearRegression.md

@@ -8,6 +8,8 @@ Consider the case of a single variable of interest y and a single predictor vari
 
 We have some data $D=\{x{\tiny i},y{\tiny i}\}$ and we assume a simple linear model of this dataset with Gaussian noise:
 
+线性回归是一种线性建模方法，这种方法用来描述自变量与一个或多个因变量的之间的关系。在只有一个因变量y和一个自变量的情况下。自变量还有以下几种叫法：协变量，输入，特征；因变量通常被叫做响应变量，输出，输出结果。
+假如我们有数据$D=\{x{\tiny i},y{\tiny i}\}$，并且假设这个数据集是满足高斯分布的线性模型：
 ```csharp
 // Prepare training Data
 var train_X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f, 7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f);
@@ -18,6 +20,8 @@ var n_samples = train_X.shape[0];
 
 Based on the given data points, we try to plot a line that models the points the best. The red line can be modelled based on the linear equation: $y = wx + b$. The motive of the linear regression algorithm is to find the best values for $w$ and $b$. Before moving on to the algorithm, le's have a look at two important concepts you must know to better understand linear regression.
 
+按照上图根据数据描述的数据点，在这些数据点之间画出一条线，这条线能达到最好模拟点的分布的效果。红色的线能够通过下面呢线性等式来描述：$y = wx + b$。线性回归算法的目标就是找到这条线对应的最好的参数$w$和$b$。在介绍线性回归算法之前，我们先看两个重要的概念，这两个概念有助于你理解线性回归算法。
+
 ### Cost Function
 
 The cost function helps us to figure out the best possible values for $w$ and $b$ which would provide the best fit line for the data points. Since we want the best values for $w$ and $b$, we convert this search problem into a minimization problem where we would like to minimize the error between the predicted value and the actual value.
@@ -65,4 +69,4 @@ When we visualize the graph in TensorBoard:
 
 ![linear-regression](_static/linear-regression-tensor-board.png)
 
-The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LinearRegression.cs).
+The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LinearRegression.cs).

docs/source/LogisticRegression.md

@@ -0,0 +1,7 @@
+# Chapter. Logistic Regression
+
+### What is logistic regression?
+
+
+
+The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LogisticRegression.cs).

docs/source/NearestNeighbor.md

@@ -0,0 +1,3 @@
+# Chapter. Nearest Neighbor
+
+The nearest neighbour algorithm was one of the first algorithms used to solve the travelling salesman problem. In it, the salesman starts at a random city and repeatedly visits the nearest city until all have been visited. It quickly yields a short tour, but usually not the optimal one.

docs/source/index.rst

@@ -26,4 +26,6 @@ Welcome to TensorFlow.NET's documentation!
    Train
    EagerMode
    LinearRegression
+   LogisticRegression
+   NearestNeighbor
    ImageRecognition

src/TensorFlowNET.Core/APIs/keras.layers.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Tensorflow.Keras;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Layers;
+
+namespace Tensorflow
+{
+    public static partial class keras
+    {
+        public static class layers
+        {
+            public static Embedding Embedding(int input_dim, int output_dim,
+                IInitializer embeddings_initializer = null,
+                bool mask_zero = false) => new Embedding(input_dim, output_dim,
+                    embeddings_initializer,
+                    mask_zero);
+
+            public static Tensor[] Input(int[] batch_shape = null,
+                TF_DataType dtype = TF_DataType.DtInvalid,
+                string name = null,
+                bool sparse = false,
+                Tensor tensor = null)
+            {
+                var batch_size = batch_shape[0];
+                var shape = batch_shape.Skip(1).ToArray();
+
+                var input_layer = new InputLayer(
+                    input_shape: shape,
+                    batch_size: batch_size,
+                    name: name,
+                    dtype: dtype,
+                    sparse: sparse,
+                    input_tensor: tensor);
+
+                var outputs = input_layer.inbound_nodes[0].output_tensors;
+
+                return outputs;
+            }
+        }
+    }
+}

src/TensorFlowNET.Core/APIs/tf.array.cs

@@ -28,7 +28,17 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <param name="conjugate"></param>
         /// <returns></returns>
-        public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false)
+        public static Tensor transpose<T1, T2>(T1 a, T2 perm, string name = "transpose", bool conjugate = false)
             => array_ops.transpose(a, perm, name, conjugate);
+
+        public static Tensor squeeze(Tensor input, int[] axis = null, string name = null, int squeeze_dims = -1)
+            => gen_array_ops.squeeze(input, axis, name);
+
+        public static Tensor one_hot(Tensor indices, int depth,
+            Tensor on_value = null,
+            Tensor off_value = null,
+            TF_DataType dtype = TF_DataType.DtInvalid,
+            int axis = -1,
+            string name = null) => array_ops.one_hot(indices, depth, dtype: dtype, axis: axis, name: name);        
     }
 }

src/TensorFlowNET.Core/APIs/tf.control.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow
+{
+    public static partial class tf
+    {
+        public static _ControlDependenciesController control_dependencies(Operation[] control_inputs) 
+            => ops.control_dependencies(control_inputs);
+    }
+}

src/TensorFlowNET.Core/APIs/tf.init.cs

@@ -10,7 +10,8 @@ namespace Tensorflow
         public static IInitializer zeros_initializer => new Zeros();
         public static IInitializer ones_initializer => new Ones();
         public static IInitializer glorot_uniform_initializer => new GlorotUniform();
-        
+        public static IInitializer uniform_initializer => new RandomUniform();
+
         public static variable_scope variable_scope(string name,
                string default_name = null,
                object values = null,
@@ -27,5 +28,13 @@ namespace Tensorflow
                   default_name,
                   values,
                   auxiliary_name_scope);
+
+        public static IInitializer truncated_normal_initializer(float mean = 0.0f,
+            float stddev = 1.0f,
+            int? seed = null,
+            TF_DataType dtype = TF_DataType.DtInvalid) => new TruncatedNormal(mean: mean,
+                stddev: stddev,
+                seed: seed,
+                dtype: dtype);
     }
 }

src/TensorFlowNET.Core/APIs/tf.layers.cs

@@ -100,6 +100,52 @@ namespace Tensorflow
 
                 return layer.apply(inputs, training: training);
             }
+
+            /// <summary>
+            /// Max pooling layer for 2D inputs (e.g. images).
+            /// </summary>
+            /// <param name="inputs">The tensor over which to pool. Must have rank 4.</param>
+            /// <param name="pool_size"></param>
+            /// <param name="strides"></param>
+            /// <param name="padding"></param>
+            /// <param name="data_format"></param>
+            /// <param name="name"></param>
+            /// <returns></returns>
+            public static Tensor max_pooling2d(Tensor inputs,
+                int[] pool_size,
+                int[] strides,
+                string padding = "valid",
+                string data_format = "channels_last",
+                string name = null)
+            {
+                var layer = new MaxPooling2D(pool_size: pool_size,
+                    strides: strides,
+                    padding: padding,
+                    data_format: data_format,
+                    name: name);
+
+                return layer.apply(inputs);
+            }
+
+            public static Tensor dense(Tensor inputs,
+                int units,
+                IActivation activation = null,
+                bool use_bias = true,
+                IInitializer kernel_initializer = null,
+                IInitializer bias_initializer = null,
+                bool trainable = true,
+                string name = null,
+                bool? reuse = null)
+            {
+                if (bias_initializer == null)
+                    bias_initializer = tf.zeros_initializer;
+
+                var layer = new Dense(units, activation, 
+                    use_bias: use_bias,
+                    kernel_initializer: kernel_initializer);
+
+                return layer.apply(inputs);
+            }
         }
     }
 }

src/TensorFlowNET.Core/APIs/tf.math.cs

@@ -6,21 +6,235 @@ namespace Tensorflow
 {
     public static partial class tf
     {
-        public static Tensor add(Tensor a, Tensor b) => gen_math_ops.add(a, b);
+        public static Tensor abs(Tensor x, string name = null)
+            => math_ops.abs(x, name);
 
-        public static Tensor sub(Tensor a, Tensor b) => gen_math_ops.sub(a, b);
+        /// <summary>
+        /// Computes acos of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor acos(Tensor x, string name = null)
+            => gen_math_ops.acos(x, name);
+
+        /// <summary>
+        /// Computes asin of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor asin(Tensor x, string name = null)
+            => gen_math_ops.asin(x, name);
+
+        public static Tensor add(Tensor a, Tensor b) 
+            => gen_math_ops.add(a, b);
+
+        /// <summary>
+        /// Computes atan of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor atan(Tensor x, string name = null)
+            => gen_math_ops.atan(x, name);
+
+        public static Tensor arg_max(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null)
+            => gen_math_ops.arg_max(input, dimension, output_type: output_type, name: name);
+
+        public static Tensor arg_min(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null)
+            => gen_math_ops.arg_min(input, dimension, output_type: output_type, name: name);
+
+        /// <summary>
+        /// Returns element-wise smallest integer not less than x.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor ceil(Tensor x, string name = null)
+            => gen_math_ops.ceil(x, name);
+
+        /// <summary>
+        /// Computes cos of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor cos(Tensor x, string name = null)
+            => gen_math_ops.cos(x, name);
+
+        /// <summary>
+        /// Computes hyperbolic cosine of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor cosh(Tensor x, string name = null)
+            => gen_math_ops.cosh(x, name);
+
+        /// <summary>
+        /// Returns element-wise largest integer not greater than x.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor floor(Tensor x, string name = null)
+            => gen_math_ops.floor(x, name);
+
+        /// <summary>
+        /// Returns the truth value of (x > y) element-wise.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor greater<Tx, Ty>(Tx x, Ty y, string name = null)
+            => gen_math_ops.greater(x, y, name);
+
+        /// <summary>
+        /// Returns the truth value of (x >= y) element-wise.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor greater_equal<Tx, Ty>(Tx x, Ty y, string name = null)
+            => gen_math_ops.greater_equal(x, y, name);
+
+        /// <summary>
+        /// Returns the truth value of (x < y) element-wise.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor less<Tx, Ty>(Tx x, Ty y, string name = null)
+            => gen_math_ops.less(x, y, name);
 
-        public static Tensor sqrt(Tensor a, string name = null) => gen_math_ops.sqrt(a, name);
+        /// <summary>
+        /// Returns the truth value of (x <= y) element-wise.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor less_equal<Tx, Ty>(Tx x, Ty y, string name = null)
+            => gen_math_ops.less_equal(x, y, name);
+
+        /// <summary>
+        /// Computes natural logarithm of (1 + x) element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor log1p(Tensor x, string name = null)
+            => gen_math_ops.log1p(x, name);
+
+        /// <summary>
+        /// Clips tensor values to a specified min and max.
+        /// </summary>
+        /// <param name="t"></param>
+        /// <param name="clip_value_min"></param>
+        /// <param name="clip_value_max"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor _clip_by_value(Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = null)
+            => gen_math_ops._clip_by_value(t, clip_value_min, clip_value_max);
+
+        public static Tensor sub(Tensor a, Tensor b) 
+            => gen_math_ops.sub(a, b);
+
+        public static Tensor sqrt(Tensor a, string name = null) 
+            => gen_math_ops.sqrt(a, name);
 
         public static Tensor subtract<T>(Tensor x, T[] y, string name = null) where T : struct
             => gen_math_ops.sub(x, ops.convert_to_tensor(y, dtype: x.dtype.as_base_dtype(), name: "y"), name);
 
-        public static Tensor multiply(Tensor x, Tensor y) => gen_math_ops.mul(x, y);
+        public static Tensor log(Tensor x, string name = null)
+            => gen_math_ops.log(x, name);
+
+        public static Tensor equal(Tensor x, Tensor y, string name = null)
+            => gen_math_ops.equal(x, y, name);
+
+        /// <summary>
+        /// Computes arctangent of `y/x` element-wise, respecting signs of the arguments.
+        /// </summary>
+        /// <param name="y"></param>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor atan2(Tensor y, Tensor x, string name = null)
+            => gen_math_ops.atan2(y, x, name);
+
+        /// <summary>
+        /// Computes the maximum of elements across dimensions of a tensor.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="input"></param>
+        /// <param name="axis"></param>
+        /// <param name="keep_dims"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor max<Tx, Ty>(Tx input, Ty axis, bool keep_dims = false, string name = null)
+            => gen_math_ops._max(input, axis, keep_dims: keep_dims, name: name);
+
+        /// <summary>
+        /// Computes the minimum of elements across dimensions of a tensor.
+        /// </summary>
+        /// <typeparam name="Tx"></typeparam>
+        /// <typeparam name="Ty"></typeparam>
+        /// <param name="input"></param>
+        /// <param name="axis"></param>
+        /// <param name="keep_dims"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor min<Tx, Ty>(Tx input, Ty axis, bool keep_dims = false, string name = null)
+            => gen_math_ops._min(input, axis, keep_dims: keep_dims, name: name);
+
+        /// <summary>
+        /// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
+        /// </summary>
+        /// <typeparam name="T1"></typeparam>
+        /// <typeparam name="T2"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor maximum<T1, T2>(T1 x, T2 y, string name = null)
+            => gen_math_ops.maximum(x, y, name: name);
+
+        /// <summary>
+        /// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+        /// </summary>
+        /// <typeparam name="T1"></typeparam>
+        /// <typeparam name="T2"></typeparam>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor minimum<T1, T2>(T1 x, T2 y, string name = null)
+            => gen_math_ops.minimum(x, y, name: name);
+
+        public static Tensor multiply(Tensor x, Tensor y) 
+            => gen_math_ops.mul(x, y);
+
+        public static Tensor negative(Tensor x, string name = null)
+            => gen_math_ops.neg(x, name);
 
         public static Tensor divide<T>(Tensor x, T[] y, string name = null) where T : struct
             => x / ops.convert_to_tensor(y, dtype: x.dtype.as_base_dtype(), name: "y");
 
-        public static Tensor pow<T1, T2>(T1 x, T2 y) => gen_math_ops.pow(x, y);
+        public static Tensor pow<T1, T2>(T1 x, T2 y) 
+            => gen_math_ops.pow(x, y);
 
         /// <summary>
         /// Computes the sum of elements across dimensions of a tensor.
@@ -28,9 +242,20 @@ namespace Tensorflow
         /// <param name="input"></param>
         /// <param name="axis"></param>
         /// <returns></returns>
-        public static Tensor reduce_sum(Tensor input, int[] axis = null) => math_ops.reduce_sum(input);
+        public static Tensor reduce_sum(Tensor input, int? axis = null, int? reduction_indices = null)
+        {
+            if(!axis.HasValue && reduction_indices.HasValue)
+                return math_ops.reduce_sum(input, reduction_indices.Value);
+            return math_ops.reduce_sum(input);
+        }
+
+        public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null, int? reduction_indices = null)
+            => math_ops.reduce_mean(input_tensor, axis: axis, keepdims: keepdims, name: name, reduction_indices: reduction_indices);
 
         public static Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) 
             => math_ops.cast(x, dtype, name);
+
+        public static Tensor argmax(Tensor input, int axis = -1, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
+            => gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);
     }
 }

src/TensorFlowNET.Core/APIs/tf.nn.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Operations;
 using Tensorflow.Operations.Activation;
 
 namespace Tensorflow
@@ -27,19 +28,39 @@ namespace Tensorflow
 
             public static IActivation relu => new relu();
 
-            public static (Tensor, Tensor, Tensor) fused_batch_norm(Tensor x,
-            RefVariable scale,
-            RefVariable offset,
-            Tensor mean = null,
-            Tensor variance = null,
-            float epsilon = 0.001f,
-            string data_format = "NHWC",
-            bool is_training = true,
-            string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance,
-                epsilon: epsilon,
-                data_format: data_format,
-                is_training: is_training,
-                name: name);
+            public static Tensor[] fused_batch_norm(Tensor x,
+                RefVariable scale,
+                RefVariable offset,
+                Tensor mean = null,
+                Tensor variance = null,
+                float epsilon = 0.001f,
+                string data_format = "NHWC",
+                bool is_training = true,
+                string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance,
+                    epsilon: epsilon,
+                    data_format: data_format,
+                    is_training: is_training,
+                    name: name);
+
+            public static IPoolFunction max_pool => new MaxPoolFunction();
+
+            public static Tensor[] top_k(Tensor input, int k = 1, bool sorted = true, string name = null)
+                => gen_nn_ops.top_kv2(input, k: k, sorted: sorted, name: name);
+
+            public static Tensor bias_add(Tensor value, RefVariable bias, string data_format = null, string name = null)
+            {
+                return Python.with(ops.name_scope(name, "BiasAdd", new { value, bias }), scope =>
+                {
+                    name = scope;
+                    return gen_nn_ops.bias_add(value, bias, data_format: data_format, name: name);
+                });
+            }
+
+            public static Tensor softmax(Tensor logits, int axis = -1, string name = null)
+                => gen_nn_ops.softmax(logits, name);
+
+            public static Tensor softmax_cross_entropy_with_logits_v2(Tensor labels, Tensor logits, int axis = -1, string name = null)
+                => nn_ops.softmax_cross_entropy_with_logits_v2_helper(labels, logits, axis: axis, name: name);
         }
     }
 }

src/TensorFlowNET.Core/APIs/tf.reshape.cs

@@ -10,5 +10,8 @@ namespace Tensorflow
                 Tensor shape,
                 string name = null) => gen_array_ops.reshape(tensor, shape, name);
 
+        public static Tensor reshape(Tensor tensor,
+                int[] shape,
+                string name = null) => gen_array_ops.reshape(tensor, shape, name);
     }
 }

src/TensorFlowNET.Core/Clustering/KMeans.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Clustering
+{
+    /// <summary>
+    /// Creates the graph for k-means clustering.
+    /// </summary>
+    public class KMeans : Python
+    {
+        public const string CLUSTERS_VAR_NAME = "clusters";
+
+        public const string SQUARED_EUCLIDEAN_DISTANCE = "squared_euclidean";
+        public const string COSINE_DISTANCE = "cosine";
+        public const string RANDOM_INIT = "random";
+        public const string KMEANS_PLUS_PLUS_INIT = "kmeans_plus_plus";
+        public const string KMC2_INIT = "kmc2";
+
+        Tensor[] _inputs;
+        int _num_clusters;
+        IInitializer _initial_clusters;
+        string _distance_metric;
+        bool _use_mini_batch;
+        int _mini_batch_steps_per_iteration;
+        int _random_seed;
+        int _kmeans_plus_plus_num_retries;
+        int _kmc2_chain_length;
+
+        public KMeans(Tensor inputs,
+            int num_clusters,
+            IInitializer initial_clusters = null,
+            string distance_metric = SQUARED_EUCLIDEAN_DISTANCE,
+            bool use_mini_batch = false,
+            int mini_batch_steps_per_iteration = 1,
+            int random_seed = 0,
+            int kmeans_plus_plus_num_retries = 2,
+            int kmc2_chain_length = 200)
+        {
+            _inputs = new Tensor[] { inputs };
+            _num_clusters = num_clusters;
+            _initial_clusters = initial_clusters;
+            _distance_metric = distance_metric;
+            _use_mini_batch = use_mini_batch;
+            _mini_batch_steps_per_iteration = mini_batch_steps_per_iteration;
+            _random_seed = random_seed;
+            _kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries;
+            _kmc2_chain_length = kmc2_chain_length;
+        }
+
+        public object training_graph()
+        {
+            var initial_clusters = _initial_clusters;
+            var num_clusters = ops.convert_to_tensor(_num_clusters);
+            var inputs = _inputs;
+            _create_variables(num_clusters);
+
+            throw new NotImplementedException("KMeans training_graph");
+        }
+
+        private RefVariable[] _create_variables(Tensor num_clusters)
+        {
+            var init_value = constant_op.constant(new float[0], dtype: TF_DataType.TF_FLOAT);
+            var cluster_centers = tf.Variable(init_value, name: CLUSTERS_VAR_NAME, validate_shape: false);
+            var cluster_centers_initialized = tf.Variable(false, dtype: TF_DataType.TF_BOOL, name: "initialized");
+            RefVariable update_in_steps = null;
+            if (_use_mini_batch && _mini_batch_steps_per_iteration > 1)
+                throw new NotImplementedException("KMeans._create_variables");
+            else
+            {
+                var cluster_centers_updated = cluster_centers;
+                var cluster_counts = _use_mini_batch ?
+                    tf.Variable(array_ops.ones(new Tensor[] { num_clusters }, dtype: TF_DataType.TF_INT64)) : 
+                    null;
+                return new RefVariable[]
+                {
+                    cluster_centers,
+                    cluster_centers_initialized,
+                    cluster_counts,
+                    cluster_centers_updated,
+                    update_in_steps
+                };
+            }
+        }
+    }
+}

src/TensorFlowNET.Core/Framework/common_shapes.py.cs

@@ -29,5 +29,15 @@ namespace Tensorflow.Framework
         {
             throw new NotFiniteNumberException();
         }
+
+        public static int? rank(Tensor tensor)
+        {
+            return tensor.rank;
+        }
+
+        public static bool has_fully_defined_shape(Tensor tensor)
+        {
+            return tensor.getShape().is_fully_defined();
+        }
     }
 }

src/TensorFlowNET.Core/Framework/meta_graph.py.cs

@@ -184,7 +184,7 @@ namespace Tensorflow
 
             // Adds graph_def or the default.
             if (graph_def == null)
-                meta_graph_def.GraphDef = graph._as_graph_def(add_shapes: true);
+                meta_graph_def.GraphDef = graph.as_graph_def(add_shapes: true);
             else
                 meta_graph_def.GraphDef = graph_def;
 

src/TensorFlowNET.Core/Framework/smart_module.cs

@@ -6,9 +6,9 @@ namespace Tensorflow.Framework
 {
     public class smart_module
     {
-        public static object smart_cond(Tensor pred, 
-            Func<(Tensor, Tensor, Tensor)> true_fn = null, 
-            Func<(Tensor, Tensor, Tensor)> false_fn = null, 
+        public static Tensor[] smart_cond<T>(Tensor pred, 
+            Func<T[]> true_fn = null, 
+            Func<T[]> false_fn = null, 
             string name = null)
         {
             return control_flow_ops.cond(pred,
@@ -17,9 +17,12 @@ namespace Tensorflow.Framework
                 name: name);
         }
 
-        public static bool smart_constant_value(Tensor pred)
+        public static bool? smart_constant_value(Tensor pred)
         {
             var pred_value = tensor_util.constant_value(pred);
+            if (pred_value is null)
+                return null;
+
             return pred_value;
         }
     }

src/TensorFlowNET.Core/Gradients/array_grad.py.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Gradients
+{
+    public class array_grad
+    {
+        public static Tensor[] _ReshapeGrad(Operation op, Tensor[] grads)
+        {
+            return new Tensor[] { array_ops.reshape(grads[0], array_ops.shape(op.inputs[0])), null };
+        }
+
+        public static Tensor[] _SqueezeGrad(Operation op, Tensor[] grads)
+        {
+            return new Tensor[] { _ReshapeToInput(op, grads[0]) };
+        }
+
+        private static Tensor _ReshapeToInput(Operation op, Tensor grad)
+        {
+            return array_ops.reshape(grad, array_ops.shape(op.inputs[0]));
+        }
+
+        public static Tensor[] _TransposeGrad(Operation op, Tensor[] grads)
+        {
+            var p = op.inputs[1];
+            return new Tensor[] { array_ops.transpose(grads[0], array_ops.invert_permutation(p)), null };
+        }
+    }
+}

src/TensorFlowNET.Core/Gradients/control_flow_grad.py.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Operations;
+
+namespace Tensorflow.Gradients
+{
+    public class control_flow_grad
+    {
+        public static Tensor[] _MergeGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var _ = grads[1];
+            var input_op = op.inputs[0].op;
+            var graph = ops.get_default_graph();
+            var op_ctxt = control_flow_util.GetOutputContext(input_op);
+            var pred = (op_ctxt as CondContext).pred;
+
+            var results = control_flow_ops._SwitchRefOrTensor(grad, pred, name: "cond_grad");
+            return new Tensor[] { results.Item1, results.Item2 };
+        }
+    }
+}

src/TensorFlowNET.Core/Gradients/gradients_impl.py.cs

@@ -131,12 +131,23 @@ namespace Tensorflow
                             // for ops that do not have gradients.
                             var grad_fn = ops.get_gradient_function(op);
 
+                            foreach(var (i, out_grad) in enumerate(out_grads))
+                            {
+                                if(out_grad == null)
+                                {
+                                    if (loop_state != null)
+                                        ;
+                                    else
+                                        out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i);
+                                }
+                            }
+
                             with(ops.name_scope(op.name + "_grad"), scope1 =>
                             {
                                 string name1 = scope1;
                                 if (grad_fn != null)
                                 {
-                                    in_grads = _MaybeCompile(grad_scope, op, out_grads[0], null, grad_fn);
+                                    in_grads = _MaybeCompile(grad_scope, op, out_grads, null, grad_fn);
                                     _VerifyGeneratedGradients(in_grads, op);
                                 }
 
@@ -226,7 +237,7 @@ namespace Tensorflow
                     $"inputs {op.inputs._inputs.Count()}");
         }
 
-        private static Tensor[] _MaybeCompile(string scope, Operation op, Tensor out_grads, Action func, Func<Operation, Tensor, Tensor[]> grad_fn)
+        private static Tensor[] _MaybeCompile(string scope, Operation op, Tensor[] out_grads, Action func, Func<Operation, Tensor[], Tensor[]> grad_fn)
         {
             scope = scope.EndsWith("/") ? scope.Substring(0, scope.Length - 1) : scope;
             return grad_fn(op, out_grads);
@@ -240,28 +251,27 @@ namespace Tensorflow
         private static Tensor[] _AggregatedGrads(Dictionary<string, Tensor[][]> grads, Operation op, string gradient_uid, object loop_state, int aggregation_method = 0)
         {
             var out_grads = _GetGrads(grads, op);
-            for(int i = 0; i < out_grads.Length; i++)
+            var return_grads = new Tensor[out_grads.Length];
+
+            foreach(var (i, out_grad) in enumerate(out_grads))
             {
-                var out_grad = out_grads[i];
-                if(loop_state != null)
+                if (loop_state != null)
                 {
 
                 }
 
-                // Grads have to be Tensors or IndexedSlices
-
                 // Aggregate multiple gradients, and convert [] to None.
-                if(out_grad != null)
+                if (out_grad != null)
                 {
-                    if(out_grad.Length < 2)
+                    if (out_grad.Length < 2)
                     {
                         string used = "nop";
-                        return new Tensor[] { out_grad[0] };
+                        return_grads[i] = out_grad[0];
                     }
                 }
             }
 
-            return null;
+            return return_grads;
         }
 
         /// <summary>

src/TensorFlowNET.Core/Gradients/math_grad.cs

@@ -0,0 +1,255 @@
+//using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Gradients
+{
+    /// <summary>
+    /// Gradients for operators defined in math_ops.py.
+    /// </summary>
+    public class math_grad : Python
+    {
+        public static Tensor[] _AddGrad(Operation op, Tensor[] grads)
+        {
+            var x = op.inputs[0];
+            var y = op.inputs[1];
+            var grad = grads[0];
+            if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad))
+                return new Tensor[] { grad, grad };
+
+            var sx = array_ops.shape(x);
+            var sy = array_ops.shape(y);
+            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
+
+            var sum1 = math_ops.reduce_sum(grad, rx);
+            var r1 = gen_array_ops.reshape(sum1, sx);
+            var sum2 = math_ops.reduce_sum(grad, ry);
+            var r2 = gen_array_ops.reshape(sum2, sy);
+
+            return new Tensor[] { r1, r2 };
+        }
+
+        public static Tensor[] _IdGrad(Operation op, Tensor[] grads)
+        {
+            return new Tensor[] { grads[0] };
+        }
+
+        public static Tensor[] _LogGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var x = op.inputs[0];
+            return with(ops.control_dependencies(new Operation[] { grad }), dp => {
+                x = math_ops.conj(x);
+                return new Tensor[] { grad * math_ops.reciprocal(x) };
+            });
+        }
+
+        public static Tensor[] _MulGrad(Operation op, Tensor[] grads)
+        {
+            var x = op.inputs[0];
+            var y = op.inputs[1];
+            var grad = grads[0];
+            if (grad is Tensor && 
+                _ShapesFullySpecifiedAndEqual(x, y, grad) &&
+                new TF_DataType[] { tf.int32, tf.float32 }.Contains(grad.dtype))
+                return new Tensor[] { gen_math_ops.mul(grad, y), gen_math_ops.mul(grad, x) };
+
+            var sx = array_ops.shape(x);
+            var sy = array_ops.shape(y);
+            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
+
+            x = math_ops.conj(x);
+            y = math_ops.conj(y);
+
+            var mul1 = gen_math_ops.mul(grad, y);
+            var reduce_sum1 = math_ops.reduce_sum(mul1, rx);
+            var reshape1 = gen_array_ops.reshape(reduce_sum1, sx);
+
+            var mul2 = gen_math_ops.mul(x, grad);
+            var reduce_sum2 = math_ops.reduce_sum(mul2, ry);
+            var reshape2 = gen_array_ops.reshape(reduce_sum2, sy);
+
+            return new Tensor[] { reshape1, reshape2 };
+        }
+
+        public static Tensor[] _MatMulGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            Tensor grad_a = null, grad_b = null;
+
+            var t_a = (bool)op.get_attr("transpose_a");
+            var t_b = (bool)op.get_attr("transpose_b");
+            var a = math_ops.conj(op.inputs[0]);
+            var b = math_ops.conj(op.inputs[1]);
+            if(!t_a && !t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b, transpose_b: true);
+                grad_b = gen_math_ops.mat_mul(a, grad, transpose_a: true);
+            }
+            else if (!t_a && t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true);
+            }
+            else if (t_a && !t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true);
+            }
+            else if (t_a && t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(b, grad, transpose_a: true, transpose_b: true);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true, transpose_b: true);
+            }
+
+            return new Tensor[] { grad_a, grad_b };
+        }
+
+        public static Tensor[] _MeanGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var sum_grad = _SumGrad(op, grads)[0];
+            var input_shape = op.inputs[0]._shape_tuple();
+            var output_shape = op.outputs[0]._shape_tuple();
+
+            var input_shape_tensor = array_ops.shape(op.inputs[0]);
+            var output_shape_tensor = array_ops.shape(op.outputs[0]);
+            var factor = _safe_shape_div(math_ops.reduce_prod(input_shape_tensor), math_ops.reduce_prod(output_shape_tensor));
+
+            return new Tensor[] { math_ops.truediv(sum_grad, math_ops.cast(factor, sum_grad.dtype)), null };
+        }
+
+        public static Tensor[] _NegGrad(Operation op, Tensor[] grads)
+        {
+            return new Tensor[] { -grads[0] };
+        }
+
+        private static Tensor _safe_shape_div(Tensor x, Tensor y)
+        {
+            return math_ops.floordiv(x, gen_math_ops.maximum(y, 1));
+        }
+
+        public static Tensor[] _SubGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var x = op.inputs[0];
+            var y = op.inputs[1];
+            if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad))
+                return new Tensor[] { grad, -grad };
+
+            var sx = array_ops.shape(x);
+            var sy = array_ops.shape(y);
+            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
+
+            var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx);
+            var r2 = gen_array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy);
+
+            return new Tensor[] { r1, r2 };
+        }
+
+        public static bool _ShapesFullySpecifiedAndEqual(Tensor x, Tensor y, Tensor grad)
+        {
+            var x_shape = x._shape_tuple();
+            var y_shape = y._shape_tuple();
+            var grad_shape = grad._shape_tuple();
+            return Enumerable.SequenceEqual(x_shape, y_shape) &&
+                Enumerable.SequenceEqual(y_shape, grad_shape) &&
+                x.NDims != -1 &&
+                !x_shape.Contains(-1);
+        }
+
+        public static Tensor[] _SumGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var input_0_shape = op.inputs[0]._shape_tuple();
+            Tensor input_shape = null;
+
+            if (input_0_shape != null)
+            {
+                var axes = tensor_util.constant_value(op.inputs[1]);
+                if(!(axes is null))
+                {
+                    var rank = input_0_shape.Length;
+                    if (Enumerable.SequenceEqual(Enumerable.Range(0, rank), axes.Data<int>()))
+                    {
+                        grad = array_ops.reshape(grad, new int[] { 1 });
+                        if (!input_0_shape.Contains(-1))
+                            input_shape = constant_op.constant(input_0_shape);
+                        else
+                            input_shape = array_ops.shape(op.inputs[0]);
+                        return new Tensor[] { gen_array_ops.tile(grad, input_shape), null };
+                    }
+                }
+            }
+
+            input_shape = array_ops.shape(op.inputs[0]);
+            ops.colocate_with(input_shape);
+            var output_shape_kept_dims = math_ops.reduced_shape(input_shape, op.inputs[1]);
+            var tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims);
+            grad = gen_array_ops.reshape(grad, output_shape_kept_dims);
+
+            return new Tensor[] { gen_array_ops.tile(grad, tile_scaling), null };
+        }
+
+        public static Tensor[] _RealDivGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var x = op.inputs[0];
+            var y = op.inputs[1];
+
+            var sx = array_ops.shape(x);
+            var sy = array_ops.shape(y);
+            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
+            x = math_ops.conj(x);
+            y = math_ops.conj(y);
+
+            var realdiv1 = gen_math_ops.real_div(-x, y);
+            var realdiv2 = gen_math_ops.real_div(realdiv1, y);
+            var reduce_sum1 = math_ops.reduce_sum(grad * realdiv2, ry);
+            var reshape1 = gen_array_ops.reshape(reduce_sum1, sy);
+            var realdiv3 = gen_math_ops.real_div(grad, y);
+            var reduce_sum2 = math_ops.reduce_sum(realdiv3, rx);
+            var reshape2 = gen_array_ops.reshape(reduce_sum2, sx);
+
+            return new Tensor[] { reshape2, reshape1 };
+        }
+
+        public static Tensor[] _PowGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var x = op.inputs[0];
+            var y = op.inputs[1];
+            var z = op.outputs[0];
+
+            var sx = array_ops.shape(x);
+            var sy = array_ops.shape(y);
+            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
+            x = math_ops.conj(x);
+            y = math_ops.conj(y);
+            z = math_ops.conj(z);
+            var pow = gen_math_ops.pow(x, y - 1.0f);
+            var mul = grad * y * pow;
+            var reduce_sum = math_ops.reduce_sum(mul, rx);
+            var gx = gen_array_ops.reshape(reduce_sum, sx);
+
+            // Avoid false singularity at x = 0
+            Tensor mask = null;
+            if (x.dtype.is_complex())
+                throw new NotImplementedException("x.dtype.is_complex()");
+            else
+                mask = x > 0.0f;
+            var ones = array_ops.ones_like(x);
+            var safe_x = array_ops.where(mask, x, ones);
+            var x1 = gen_array_ops.log(safe_x);
+            var y1 = array_ops.zeros_like(x);
+            var log_x = array_ops.where(mask, x1, y1);
+            var mul1 = grad * z * log_x;
+            var reduce_sum1 = math_ops.reduce_sum(mul1, ry);
+            var gy = gen_array_ops.reshape(reduce_sum1, sy);
+
+            return new Tensor[] { gx, gy };
+        }
+    }
+}

src/TensorFlowNET.Core/Gradients/math_grad.py.cs

@@ -1,160 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Tensorflow
-{
-    /// <summary>
-    /// Gradients for operators defined in math_ops.py.
-    /// </summary>
-    public class math_grad
-    {
-        public static (Tensor, Tensor) _AddGrad(Operation op, Tensor grad)
-        {
-            var x = op.inputs[0];
-            var y = op.inputs[1];
-            if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad))
-                return (grad, grad);
-
-            var sx = array_ops.shape(x);
-            var sy = array_ops.shape(y);
-            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
-
-            var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx);
-            var r2 = gen_array_ops.reshape(math_ops.reduce_sum(grad, ry), sy);
-
-            return (r1, r2);
-        }
-
-        public static Tensor _IdGrad(Operation op, Tensor grad)
-        {
-            return grad;
-        }
-
-        public static (Tensor, Tensor) _MulGrad(Operation op, Tensor grad)
-        {
-            var x = op.inputs[0];
-            var y = op.inputs[1];
-            if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad) &&
-                new TF_DataType[] { tf.int32, tf.float32 }.Contains(grad.dtype))
-                return (gen_math_ops.mul(grad, y), gen_math_ops.mul(grad, x));
-
-            var sx = array_ops.shape(x);
-            var sy = array_ops.shape(y);
-            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
-
-            x = math_ops.conj(x);
-            y = math_ops.conj(y);
-
-            var mul1 = gen_math_ops.mul(grad, y);
-            var mul2 = gen_math_ops.mul(x, grad);
-            var reduce_sum1 = math_ops.reduce_sum(mul1, rx);
-            var reduce_sum2 = math_ops.reduce_sum(mul2, ry);
-            var reshape1 = gen_array_ops.reshape(reduce_sum1, sx);
-            var reshape2 = gen_array_ops.reshape(reduce_sum2, sy);
-
-            return (reshape1, reshape2);
-        }
-
-        public static (Tensor, Tensor) _SubGrad(Operation op, Tensor grad)
-        {
-            var x = op.inputs[0];
-            var y = op.inputs[1];
-            if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad))
-                return (grad, -grad);
-
-            var sx = array_ops.shape(x);
-            var sy = array_ops.shape(y);
-            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
-
-            var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx);
-            var r2 = gen_array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy);
-
-            return (r1, r2);
-        }
-
-        public static bool _ShapesFullySpecifiedAndEqual(Tensor x, Tensor y, Tensor grad)
-        {
-            return x.NDims == y.NDims && y.NDims == grad.NDims && x.NDims > -1;
-        }
-
-        public static (Tensor, Tensor) _SumGrad(Operation op, Tensor grad)
-        {
-            if (op.inputs[0].NDims > -1)
-            {
-
-            }
-
-            var input_shape = array_ops.shape(op.inputs[0]);
-            ops.colocate_with(input_shape);
-            var output_shape_kept_dims = math_ops.reduced_shape(input_shape, op.inputs[1]);
-            var tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims);
-            grad = gen_array_ops.reshape(grad, output_shape_kept_dims);
-
-            return (gen_array_ops.tile(grad, tile_scaling), null);
-        }
-
-        public static Tensor _safe_shape_div(Tensor x, Tensor y)
-        {
-            return math_ops.floordiv(x, gen_math_ops.maximum(y, 1));
-        }
-
-        public static (Tensor, Tensor) _RealDivGrad(Operation op, Tensor grad)
-        {
-            var x = op.inputs[0];
-            var y = op.inputs[1];
-
-            var sx = array_ops.shape(x);
-            var sy = array_ops.shape(y);
-            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
-            x = math_ops.conj(x);
-            y = math_ops.conj(y);
-
-            var realdiv1 = gen_math_ops.real_div(-x, y);
-            var realdiv2 = gen_math_ops.real_div(realdiv1, y);
-            var reduce_sum1 = math_ops.reduce_sum(grad * realdiv2, ry);
-            var reshape1 = gen_array_ops.reshape(reduce_sum1, sy);
-            var realdiv3 = gen_math_ops.real_div(grad, y);
-            var reduce_sum2 = math_ops.reduce_sum(realdiv3, rx);
-            var reshape2 = gen_array_ops.reshape(reduce_sum2, sx);
-
-            return (reshape2, reshape1);
-        }
-
-        public static (Tensor, Tensor) _PowGrad(Operation op, Tensor grad)
-        {
-            var x = op.inputs[0];
-            var y = op.inputs[1];
-            var z = op.outputs[0];
-
-            var sx = array_ops.shape(x);
-            var sy = array_ops.shape(y);
-            var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
-            x = math_ops.conj(x);
-            y = math_ops.conj(y);
-            z = math_ops.conj(z);
-            var pow = gen_math_ops.pow(x, y - 1.0f);
-            var mul = grad * y * pow;
-            var reduce_sum = math_ops.reduce_sum(mul, rx);
-            var gx = gen_array_ops.reshape(reduce_sum, sx);
-
-            // Avoid false singularity at x = 0
-            Tensor mask = null;
-            if (x.dtype.is_complex())
-                throw new NotImplementedException("x.dtype.is_complex()");
-            else
-                mask = x > 0.0f;
-            var ones = array_ops.ones_like(x);
-            var safe_x = array_ops.where(mask, x, ones);
-            var x1 = gen_array_ops.log(safe_x);
-            var y1 = array_ops.zeros_like(x);
-            var log_x = array_ops.where(mask, x1, y1);
-            var mul1 = grad * z * log_x;
-            var reduce_sum1 = math_ops.reduce_sum(mul1, ry);
-            var gy = gen_array_ops.reshape(reduce_sum1, sy);
-
-            return (gx, gy);
-        }
-    }
-}

src/TensorFlowNET.Core/Gradients/nn_grad.py.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Tensorflow.Operations;
+
+namespace Tensorflow.Gradients
+{
+    public class nn_grad
+    {
+        /// <summary>
+        /// Return the gradients for the 2 inputs of bias_op.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <param name="grad"></param>
+        /// <returns></returns>
+        public static Tensor[] _BiasAddGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            string data_format = op.get_attr("data_format")?.ToString();
+            var bias_add_grad = gen_nn_ops.bias_add_grad(out_backprop: grad, data_format: data_format);
+            return new Tensor[] { grad, bias_add_grad };
+        }
+
+        public static Tensor[] _ReluGrad(Operation op, Tensor[] grads)
+        {
+            return new Tensor[] { gen_nn_ops.relu_grad(grads[0], op.outputs[0]) };
+        }
+
+        /// <summary>
+        /// The derivative of the softmax nonlinearity.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <param name="grads"></param>
+        /// <returns></returns>
+        public static Tensor[] _SoftmaxGrad(Operation op, Tensor[] grads)
+        {
+            var grad_softmax = grads[0];
+
+            var softmax = op.outputs[0];
+            var mul = grad_softmax * softmax;
+            var sum_channels = math_ops.reduce_sum(mul, -1, keepdims: true);
+            var sub = grad_softmax - sum_channels;
+            return new Tensor[] { sub * softmax };
+        }
+
+        /// <summary>
+        /// Gradient function for SoftmaxCrossEntropyWithLogits.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <param name="grad_loss"></param>
+        /// <param name="grad_grad"></param>
+        /// <returns></returns>
+        public static Tensor[] _SoftmaxCrossEntropyWithLogitsGrad(Operation op, Tensor[] grads)
+        {
+            var grad_loss = grads[0];
+            var grad_grad = grads[1];
+            var softmax_grad = op.outputs[1];
+            var grad = _BroadcastMul(grad_loss, softmax_grad);
+
+            var logits = op.inputs[0];
+            if(grad_grad != null && !IsZero(grad_grad))
+            {
+                throw new NotImplementedException("_SoftmaxCrossEntropyWithLogitsGrad");
+            }
+
+            return new Tensor[] 
+            {
+                grad,
+                _BroadcastMul(grad_loss, -nn_ops.log_softmax(logits))
+            };
+        }
+
+        private static bool IsZero(Tensor g)
+        {
+            if (new string[] { "ZerosLike", "Zeros" }.Contains(g.op.type))
+                return true;
+
+            throw new NotImplementedException("IsZero");
+        }
+
+        private static Tensor _BroadcastMul(Tensor vec, Tensor mat)
+        {
+            vec = array_ops.expand_dims(vec, -1);
+            return vec * mat;
+        }
+
+        /// <summary>
+        /// Return the gradients for TopK.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <param name="grads"></param>
+        /// <returns></returns>
+        public static Tensor[] _TopKGrad(Operation op, Tensor[] grads)
+        {
+            var grad = grads[0];
+            var _ = grads[1];
+
+            var in_shape = array_ops.shape(op.inputs[0]);
+            var ind_shape = array_ops.shape(op.outputs[1]);
+
+            // int32 is not supported on GPU hence up-casting
+            var cast = math_ops.cast(ind_shape, TF_DataType.TF_INT64);
+            var size = array_ops.size(ind_shape) - 1;
+            var ind_lastdim = array_ops.gather(cast, size);
+
+            // Flatten indices to 2D.
+            var stack = array_ops.stack(new object[] { -1L, ind_lastdim });
+            var ind_2d = array_ops.reshape(op.outputs[1], stack);
+
+            var in_lastdim = array_ops.gather(math_ops.cast(in_shape, TF_DataType.TF_INT64), 
+                array_ops.size(in_shape) - 1);
+            var outerdim = array_ops.shape(ind_2d)[0];
+
+            // Compute linear indices(flattened to 1D).
+            var cast1 = math_ops.cast(outerdim, TF_DataType.TF_INT64);
+            var range2 = math_ops.range(0L, cast1 * in_lastdim, in_lastdim);
+            var dim2 = array_ops.expand_dims(range2, -1);
+            var cast2 = math_ops.cast(dim2, TF_DataType.TF_INT32);
+            var ind = array_ops.reshape(ind_2d + cast2, new int[] { -1 });
+
+            // Substitute grad to appropriate locations and fill the rest with zeros,
+            // finally reshaping it to the original input shape.
+            var scatter = gen_array_ops.scatter_nd(array_ops.expand_dims(ind, -1),
+              array_ops.reshape(grad, new int[] { -1 }),
+              new Tensor[] { math_ops.reduce_prod(in_shape) });
+
+            return new Tensor[]
+            {
+                array_ops.reshape(scatter, in_shape),
+                array_ops.zeros(new int[0], dtype: TF_DataType.TF_INT32)
+            };
+        }
+    }
+}

src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Gradients;
+
+namespace Tensorflow
+{
+    public partial class ops
+    {
+        public static Func<Operation, Tensor[], Tensor[]> get_gradient_function(Operation op)
+        {
+            if (op.inputs == null) return null;
+
+            // map tensorflow\python\ops\math_grad.py
+            return (oper, out_grads) =>
+            {
+                // Console.WriteLine($"get_gradient_function: {oper.type} '{oper.name}'");
+
+                switch (oper.type)
+                {
+                    case "Add":
+                        return math_grad._AddGrad(oper, out_grads);
+                    case "BiasAdd":
+                        return nn_grad._BiasAddGrad(oper, out_grads);
+                    case "Identity":
+                        return math_grad._IdGrad(oper, out_grads);
+                    case "Log":
+                        return math_grad._LogGrad(oper, out_grads);
+                    case "MatMul":
+                        return math_grad._MatMulGrad(oper, out_grads);
+                    case "Merge":
+                        return control_flow_grad._MergeGrad(oper, out_grads);
+                    case "Mul":
+                        return math_grad._MulGrad(oper, out_grads);
+                    case "Mean":
+                        return math_grad._MeanGrad(oper, out_grads);
+                    case "Neg":
+                        return math_grad._NegGrad(oper, out_grads);
+                    case "Sum":
+                        return math_grad._SumGrad(oper, out_grads);
+                    case "Sub":
+                        return math_grad._SubGrad(oper, out_grads);
+                    case "Pow":
+                        return math_grad._PowGrad(oper, out_grads);
+                    case "RealDiv":
+                        return math_grad._RealDivGrad(oper, out_grads);
+                    case "Reshape":
+                        return array_grad._ReshapeGrad(oper, out_grads);
+                    case "Relu":
+                        return nn_grad._ReluGrad(oper, out_grads);
+                    case "Squeeze":
+                        return array_grad._SqueezeGrad(oper, out_grads);
+                    case "Softmax":
+                        return nn_grad._SoftmaxGrad(oper, out_grads);
+                    case "SoftmaxCrossEntropyWithLogits":
+                        return nn_grad._SoftmaxCrossEntropyWithLogitsGrad(oper, out_grads);
+                    case "Transpose":
+                        return array_grad._TransposeGrad(oper, out_grads);
+                    case "TopK":
+                    case "TopKV2":
+                        return nn_grad._TopKGrad(oper, out_grads);
+                    default:
+                        throw new NotImplementedException($"get_gradient_function {oper.type}");
+                }
+            };
+        }
+    }
+}

src/TensorFlowNET.Core/Graphs/FreezeGraph.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow
+{
+    public class FreezeGraph
+    {
+        public static void freeze_graph(string input_graph,
+                              string input_saver,
+                              bool input_binary,
+                              string input_checkpoint,
+                              string output_node_names,
+                              string restore_op_name,
+                              string filename_tensor_name,
+                              string output_graph,
+                              bool clear_devices,
+                              string initializer_nodes)
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Graphs/Graph.Export.cs

@@ -18,7 +18,7 @@ namespace Tensorflow
             return buffer;
         }
 
-        public GraphDef _as_graph_def(bool add_shapes = false)
+        private GraphDef _as_graph_def(bool add_shapes = false)
         {
             var buffer = ToGraphDef(Status);
             Status.Check();
@@ -30,5 +30,8 @@ namespace Tensorflow
 
             return def;
         }
+
+        public GraphDef as_graph_def(bool add_shapes = false) 
+            => _as_graph_def(add_shapes);
     }
 }

src/TensorFlowNET.Core/Graphs/Graph.cs

@@ -355,7 +355,7 @@ namespace Tensorflow
             return _collections.Keys.Where(x => !x.StartsWith("__")).ToArray();
         }
 
-        public object get_collection(string name, string scope = "")
+        public object get_collection(string name, string scope = null)
         {
             return _collections.ContainsKey(name) ? _collections[name] : null;
         }

src/TensorFlowNET.Core/Graphs/graph_io.py.cs

@@ -10,7 +10,7 @@ namespace Tensorflow
     {
         public static string write_graph(Graph graph, string logdir, string name, bool as_text = true)
         {
-            var graph_def = graph._as_graph_def();
+            var graph_def = graph.as_graph_def();
             string path = Path.Combine(logdir, name);
             if (as_text)
                 File.WriteAllText(path, graph_def.ToString());

src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs

@@ -9,17 +9,20 @@ namespace Tensorflow.Keras.Engine
     /// </summary>
     public class InputSpec
     {
-        public int ndim;
+        public int? ndim;
+        public int? min_ndim;
         Dictionary<int, int> axes;
 
-        public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, 
+        public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid,
             int? ndim = null,
+            int? min_ndim = null,
             Dictionary<int, int> axes = null)
         {
-            this.ndim = ndim.Value;
+            this.ndim = ndim;
             if (axes == null)
                 axes = new Dictionary<int, int>();
             this.axes = axes;
+            this.min_ndim = min_ndim;
         }
     }
 }

src/TensorFlowNET.Core/Keras/Engine/Model.cs

@@ -4,7 +4,12 @@ using System.Text;
 
 namespace Tensorflow.Keras.Engine
 {
-    internal class Model : Network
+    public class Model : Network
     {
+        public Model(string name = null) 
+            : base(name: name)
+        {
+
+        }
     }
 }

src/TensorFlowNET.Core/Keras/Engine/Network.cs

@@ -1,10 +1,41 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Keras.Layers;
 
 namespace Tensorflow.Keras.Engine
 {
     public class Network : Layer
     {
+        protected bool _is_compiled;
+        protected bool _expects_training_arg;
+        protected bool _compute_output_and_mask_jointly;
+        /// <summary>
+        /// All layers in order of horizontal graph traversal.
+        /// Entries are unique. Includes input and output layers.
+        /// </summary>
+        protected List<Layer> _layers;
+
+        public Network(string name = null) 
+            : base(name: name)
+        {
+            _init_subclassed_network(name);
+        }
+
+        protected virtual void _init_subclassed_network(string name = null)
+        {
+            _base_init(name: name);
+        }
+
+        protected virtual void _base_init(string name = null)
+        {
+            _init_set_name(name);
+            trainable = true;
+            _is_compiled = false;
+            _expects_training_arg = false;
+            _compute_output_and_mask_jointly = false;
+            supports_masking = false;
+            _layers = new List<Layer>();
+        }
     }
 }

src/TensorFlowNET.Core/Keras/Engine/Sequential.cs

@@ -1,24 +1,56 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Keras.Layers;
 
 namespace Tensorflow.Keras.Engine
 {
-    public class Sequential : Network, IPython
+    public class Sequential : Model, IPython
     {
-        public void Dispose()
+        public Sequential(string name = null) 
+            : base(name: name)
         {
-            throw new NotImplementedException();
+            supports_masking = true;
+            _compute_output_and_mask_jointly = true;
         }
 
         public void __enter__()
         {
-            throw new NotImplementedException();
+            
+        }
+
+        public void add(Layer layer)
+        {
+            built = false;
+            var set_inputs = false;
+            if(_layers.Count == 0)
+            {
+                var (batch_shape, dtype) = (layer._batch_input_shape, layer._dtype);
+                if(batch_shape != null)
+                {
+                    // Instantiate an input layer.
+                    var x = keras.layers.Input(
+                      batch_shape: batch_shape,
+                      dtype: dtype,
+                      name: layer._name + "_input");
+
+                    // This will build the current layer
+                    // and create the node connecting the current layer
+                    // to the input layer we just created.
+                    layer.__call__(x);
+                    set_inputs = true;
+                }
+            }
         }
 
         public void __exit__()
         {
-            throw new NotImplementedException();
+            
+        }
+
+        public void Dispose()
+        {
+
         }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs

@@ -3,11 +3,10 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using Tensorflow.Keras.Utils;
-using Tensorflow.Layers;
 
 namespace Tensorflow.Keras.Layers
 {
-    public class BatchNormalization : Layer
+    public class BatchNormalization : Tensorflow.Layers.Layer
     {
         private bool _USE_V2_BEHAVIOR = true;
         private float momentum;
@@ -132,6 +131,7 @@ namespace Tensorflow.Keras.Layers
             if (fused)
             {
                 outputs = _fused_batch_norm(inputs, training: training);
+                return outputs;
             }
 
             throw new NotImplementedException("BatchNormalization call");
@@ -142,7 +142,7 @@ namespace Tensorflow.Keras.Layers
             var beta = this.beta;
             var gamma = this.gamma;
 
-            Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_training = () =>
+            Func<Tensor[]> _fused_batch_norm_training = () =>
             {
                 return tf.nn.fused_batch_norm(
                   inputs,
@@ -152,7 +152,7 @@ namespace Tensorflow.Keras.Layers
                   data_format: _data_format);
             };
 
-            Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_inference = () =>
+            Func<Tensor[]> _fused_batch_norm_inference = () =>
             {
                 return tf.nn.fused_batch_norm(
                   inputs,
@@ -165,9 +165,41 @@ namespace Tensorflow.Keras.Layers
                   data_format: _data_format);
             };
 
-            tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference);
+            var results = tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference);
+            var (output, mean, variance) = (results[0], results[1], results[2]);
+            var training_value = tf_utils.constant_value(training);
 
-            throw new NotImplementedException("_fused_batch_norm");
+            Tensor momentum_tensor;
+            if (training_value == null)
+            {
+                momentum_tensor = tf_utils.smart_cond(training,
+                    () => new float[] { momentum }, () => new float[] { 1.0f })[0];
+            }
+            else
+            {
+                momentum_tensor = ops.convert_to_tensor(momentum);
+            }
+                
+            if(training_value == null)
+            {
+                var mean_update = _assign_moving_average(moving_mean, mean, momentum_tensor);
+                var variance_update = _assign_moving_average(moving_variance, variance, momentum_tensor);
+                add_update(new Tensor[] { mean_update }, inputs: true);
+                add_update(new Tensor[] { variance_update }, inputs: true);
+            }
+
+            return output;
+        }
+
+        public Tensor _assign_moving_average(RefVariable variable, Tensor value, Tensor momentum)
+        {
+            return Python.with(ops.name_scope(null, "AssignMovingAvg", new { variable, value, momentum }), scope =>
+            {
+                // var cm = ops.colocate_with(variable);
+                var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay");
+                var update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay;
+                return state_ops.assign_sub(variable, update_delta, name: scope);
+            });
         }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/Dense.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Operations.Activation;
+using static Tensorflow.tf;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class Dense : Tensorflow.Layers.Layer
+    {
+        protected int units;
+        protected IActivation activation;
+        protected bool use_bias;
+        protected IInitializer kernel_initializer;
+        protected IInitializer bias_initializer;
+        protected RefVariable kernel;
+        protected RefVariable bias;
+
+        public Dense(int units,
+            IActivation activation,
+            bool use_bias = true,
+            bool trainable = false,
+            IInitializer kernel_initializer = null,
+            IInitializer bias_initializer = null) : base(trainable: trainable)
+        {
+            this.units = units;
+            this.activation = activation;
+            this.use_bias = use_bias;
+            this.kernel_initializer = kernel_initializer;
+            this.bias_initializer = bias_initializer;
+            this.supports_masking = true;
+            this.input_spec = new InputSpec(min_ndim: 2);
+        }
+
+        protected override void build(TensorShape input_shape)
+        {
+            var last_dim = input_shape.Dimensions.Last();
+            var axes = new Dictionary<int, int>();
+            axes[-1] = last_dim;
+            input_spec = new InputSpec(min_ndim: 2, axes: axes);
+            kernel = add_weight(
+                "kernel",
+                shape: new int[] { last_dim, units },
+                initializer: kernel_initializer,
+                dtype: _dtype,
+                trainable: true);
+            if (use_bias)
+                bias = add_weight(
+                  "bias",
+                  shape: new int[] { units },
+                  initializer: bias_initializer,
+                  dtype: _dtype,
+                  trainable: true);
+
+            built = true;
+        }
+
+        protected override Tensor call(Tensor inputs, Tensor training = null)
+        {
+            Tensor outputs = null;
+            var rank = inputs.rank;
+            if(rank > 2)
+            {
+                throw new NotImplementedException("call rank > 2");
+            }
+            else
+            {
+                outputs = gen_math_ops.mat_mul(inputs, kernel);
+            }
+
+            if (use_bias)
+                outputs = nn.bias_add(outputs, bias);
+            if (activation != null)
+                return activation.Activate(outputs);
+
+            return outputs;
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/Embedding.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class Embedding : Layer
+    {
+        private int input_dim;
+        private int output_dim;
+        private bool mask_zero;
+        public RefVariable embeddings;
+        public IInitializer embeddings_initializer;
+
+        public Embedding(int input_dim, int output_dim,
+            IInitializer embeddings_initializer = null,
+            bool mask_zero = false,
+            TF_DataType dtype = TF_DataType.TF_FLOAT,
+            int[] input_shape = null) : base(dtype: dtype, input_shape: input_shape)
+        {
+            this.input_dim = input_dim;
+            this.output_dim = output_dim;
+            this.embeddings_initializer = embeddings_initializer == null ? tf.uniform_initializer : embeddings_initializer;
+            this.mask_zero = mask_zero;
+            supports_masking = mask_zero;
+        }
+
+        protected override void build(TensorShape input_shape)
+        {
+            embeddings = add_weight(shape: new int[] { input_dim, output_dim },
+                initializer: embeddings_initializer,
+                name: "embeddings");
+            built = true;
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/IPoolFunction.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow
+{
+    public interface IPoolFunction
+    {
+        Tensor Apply(Tensor value,
+            int[] ksize,
+            int[] strides,
+            string padding,
+            string data_format = "NHWC",
+            string name = null);
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Layers
+{
+    /// <summary>
+    /// Layer to be used as an entry point into a Network (a graph of layers).
+    /// </summary>
+    public class InputLayer : Layer
+    {
+        public bool sparse;
+        public int? batch_size;
+        public bool is_placeholder;
+
+        public InputLayer(int[] input_shape = null,
+            int? batch_size = null,
+            TF_DataType dtype = TF_DataType.DtInvalid,
+            string name = null,
+            bool sparse = false,
+            Tensor input_tensor = null)
+        {
+            built = true;
+            this.sparse = sparse;
+            this.batch_size = batch_size;
+            this.supports_masking = true;
+
+            if (input_tensor == null)
+            {
+                var batch_input_shape = new int[] { batch_size.HasValue ? batch_size.Value : -1, -1 };
+
+                if (sparse)
+                {
+                    throw new NotImplementedException("InputLayer sparse is true");
+                }
+                else
+                {
+                    input_tensor = backend.placeholder(
+                          shape: batch_input_shape,
+                          dtype: dtype,
+                          name: name);
+                }
+
+                is_placeholder = true;
+                _batch_input_shape = batch_input_shape;
+            }
+
+            new Node(this,
+                inbound_layers: new Layer[0],
+                node_indices: new int[0],
+                tensor_indices: new int[0],
+                input_tensors: new Tensor[] { input_tensor },
+                output_tensors: new Tensor[] { input_tensor });
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Engine/Layer.cs → src/TensorFlowNET.Core/Keras/Layers/Layer.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
+using Tensorflow.Keras.Engine;
 using Tensorflow.Keras.Utils;
 
-namespace Tensorflow.Keras.Engine
+namespace Tensorflow.Keras.Layers
 {
     /// <summary>
     /// Base layer class.
@@ -19,7 +21,7 @@ namespace Tensorflow.Keras.Engine
         /// </summary>
         protected bool built;
         protected bool trainable;
-        protected TF_DataType _dtype;
+        public TF_DataType _dtype;
         /// <summary>
         /// A stateful layer is a layer whose updates are run during inference too,
         /// for instance stateful RNNs.
@@ -31,11 +33,22 @@ namespace Tensorflow.Keras.Engine
         protected InputSpec input_spec;
         protected bool supports_masking;
         protected List<RefVariable> _trainable_weights;
-        protected string _name;
+        public string _name;
         protected string _base_name;
         protected bool _compute_previous_mask;
+        protected List<Operation> _updates;
+        public int[] _batch_input_shape;
 
-        public Layer(bool trainable = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid)
+        private List<Node> _inbound_nodes;
+        public List<Node> inbound_nodes => _inbound_nodes;
+
+        private List<Node> _outbound_nodes;
+        public List<Node> outbound_nodes => _outbound_nodes;
+
+        public Layer(bool trainable = true, 
+            string name = null, 
+            TF_DataType dtype = TF_DataType.DtInvalid,
+            int[] input_shape = null)
         {
             this.trainable = trainable;
             this._dtype = dtype;
@@ -45,13 +58,22 @@ namespace Tensorflow.Keras.Engine
             _init_set_name(name);
             _trainable_weights = new List<RefVariable>();
             _compute_previous_mask = false;
+            _updates = new List<Operation>();
+
+            // Manage input shape information if passed.
+
+            _batch_input_shape = new int[] { -1, -1 };
+
+            _dtype = dtype;
+
+            _inbound_nodes = new List<Node>();
         }
 
-        public Tensor __call__(Tensor inputs,
+        public Tensor __call__(Tensor[] inputs,
             Tensor training = null,
             VariableScope scope = null)
         {
-            var input_list = new Tensor[] { inputs };
+            var input_list = inputs;
             Tensor outputs = null;
 
             // We will attempt to build a TF graph if & only if all inputs are symbolic.
@@ -74,9 +96,9 @@ namespace Tensorflow.Keras.Engine
                     // Symbolic execution on symbolic tensors. We will attempt to build
                     // the corresponding TF subgraph inside `backend.get_graph()`
                     var graph = backend.get_graph();
-                    outputs = call(inputs, training: training);
-                    _handle_activity_regularization(inputs, outputs);
-                    _set_mask_metadata(inputs, outputs, null);
+                    outputs = call(inputs[0], training: training);
+                    _handle_activity_regularization(inputs[0], outputs);
+                    _set_mask_metadata(inputs[0], outputs, null);
                 }
             });
 
@@ -103,7 +125,7 @@ namespace Tensorflow.Keras.Engine
 
         protected virtual Tensor call(Tensor inputs, Tensor training = null)
         {
-            throw new NotImplementedException("Layer.call");
+            return inputs;
         }
 
         protected virtual string _name_scope()
@@ -111,15 +133,15 @@ namespace Tensorflow.Keras.Engine
             return null;
         }
 
-        protected void _maybe_build(Tensor inputs)
+        protected void _maybe_build(Tensor[] inputs)
         {
-            var input_list = new Tensor[] { inputs };
-            build(inputs.getShape());
+            var input_list = inputs;
+            build(input_list[0].getShape());
         }
 
         protected virtual void build(TensorShape input_shape)
         {
-            throw new NotImplementedException("Layer.build");
+            built = true;
         }
 
         protected virtual RefVariable add_weight(string name,
@@ -129,10 +151,16 @@ namespace Tensorflow.Keras.Engine
             bool? trainable = null,
             Func<string, int[], TF_DataType, IInitializer, bool, RefVariable> getter = null)
         {
+            if (dtype == TF_DataType.DtInvalid)
+                dtype = TF_DataType.TF_FLOAT;
+
+            if (trainable == null)
+                trainable = true;
+
             var variable = _add_variable_with_custom_getter(name,
                 shape,
                 dtype: dtype,
-                getter: getter,
+                getter: getter == null ? base_layer_utils.make_variable : getter,
                 overwrite: true,
                 initializer: initializer,
                 trainable: trainable.Value);
@@ -142,6 +170,12 @@ namespace Tensorflow.Keras.Engine
             return variable;
         }
 
+        protected virtual void add_update(Tensor[] updates, bool inputs = false)
+        {
+            var updates_op = updates.Select(x => x.op).ToArray();
+            _updates.AddRange(updates_op);
+        }
+
         protected virtual void _init_set_name(string name)
         {
             string base_name = name;

src/TensorFlowNET.Core/Keras/Layers/MaxPooling2D.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.tf;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class MaxPooling2D : Pooling2D
+    {
+        public MaxPooling2D(
+            int[] pool_size,
+            int[] strides,
+            string padding = "valid",
+            string data_format = null,
+            string name = null) : base(nn.max_pool, pool_size,
+                strides,
+                padding: padding,
+                data_format: data_format,
+                name: name)
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/Node.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Keras.Layers
+{
+    /// <summary>
+    /// A `Node` describes the connectivity between two layers.
+    /// </summary>
+    public class Node
+    {
+        public InputLayer outbound_layer;
+        public Layer[] inbound_layers;
+        public int[] node_indices;
+        public int[] tensor_indices;
+        public Tensor[] input_tensors;
+        public Tensor[] output_tensors;
+        public int[][] input_shapes;
+        public int[][] output_shapes;
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="outbound_layer">
+        /// the layer that takes
+        /// `input_tensors` and turns them into `output_tensors`
+        /// (the node gets created when the `call`
+        /// method of the layer was called).
+        /// </param>
+        /// <param name="inbound_layers">
+        /// a list of layers, the same length as `input_tensors`,
+        /// the layers from where `input_tensors` originate.
+        /// </param>
+        /// <param name="node_indices">
+        /// a list of integers, the same length as `inbound_layers`.
+        /// `node_indices[i]` is the origin node of `input_tensors[i]`
+        /// (necessary since each inbound layer might have several nodes,
+        /// e.g. if the layer is being shared with a different data stream).
+        /// </param>
+        /// <param name="tensor_indices"></param>
+        /// <param name="input_tensors">list of input tensors.</param>
+        /// <param name="output_tensors">list of output tensors.</param>
+        public Node(InputLayer outbound_layer, 
+            Layer[] inbound_layers, 
+            int[] node_indices,
+            int[] tensor_indices,
+            Tensor[] input_tensors,
+            Tensor[] output_tensors)
+        {
+            this.outbound_layer = outbound_layer;
+            this.inbound_layers = inbound_layers;
+            this.node_indices = node_indices;
+            this.tensor_indices = tensor_indices;
+            this.input_tensors = input_tensors;
+            this.output_tensors = output_tensors;
+
+            input_shapes = input_tensors.Select(x => x._shape_tuple()).ToArray();
+            output_shapes = output_tensors.Select(x => x._shape_tuple()).ToArray();
+
+            // Add nodes to all layers involved.
+            foreach (var layer in inbound_layers)
+            {
+                if (layer != null)
+                    layer.outbound_nodes.Add(this);
+            }
+
+            outbound_layer.inbound_nodes.Add(this);
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Utils;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class Pooling2D : Tensorflow.Layers.Layer
+    {
+        private IPoolFunction pool_function;
+        private int[] pool_size;
+        private int[] strides;
+        private string padding;
+        private string data_format;
+        private InputSpec input_spec;
+
+        public Pooling2D(IPoolFunction pool_function,
+            int[] pool_size,
+            int[] strides,
+            string padding = "valid",
+            string data_format = null,
+            string name = null) : base(name: name)
+        {
+            this.pool_function = pool_function;
+            this.pool_size = conv_utils.normalize_tuple(pool_size, 2, "pool_size");
+            this.strides = conv_utils.normalize_tuple(strides, 2, "strides");
+            this.padding = conv_utils.normalize_padding(padding);
+            this.data_format = conv_utils.normalize_data_format(data_format);
+            this.input_spec = new InputSpec(ndim: 4);
+        }
+
+        protected override Tensor call(Tensor inputs, Tensor training = null)
+        {
+            int[] pool_shape;
+            if (data_format == "channels_last")
+            {
+                pool_shape = new int[] { 1, pool_size[0], pool_size[1], 1 };
+                strides = new int[] { 1, strides[0], strides[1], 1 };
+            }
+            else
+            {
+                pool_shape = new int[] { 1, 1, pool_size[0], pool_size[1] };
+                strides = new int[] { 1, 1, strides[0], strides[1] };
+            }
+
+            var outputs = pool_function.Apply(
+                inputs,
+                ksize: pool_shape,
+                strides: strides,
+                padding: padding.ToUpper(),
+                data_format: conv_utils.convert_data_format(data_format, 4));
+
+            return outputs;
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs → src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs

@@ -2,10 +2,19 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace Tensorflow.Keras.Engine
+namespace Tensorflow.Keras.Utils
 {
     public class base_layer_utils
     {
+        public static RefVariable make_variable(string name,
+            int[] shape,
+            TF_DataType dtype = TF_DataType.TF_FLOAT,
+            IInitializer initializer = null,
+            bool trainable = false)
+        {
+            throw new NotImplementedException("");
+        }
+
         /// <summary>
         /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph.
         /// </summary>

src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs

@@ -29,5 +29,20 @@ namespace Tensorflow.Keras.Utils
             else
                 throw new ValueError($"Invalid data_format: {data_format}");
         }
+
+        public static int[] normalize_tuple(int[] value, int n, string name)
+        {
+            return value;
+        }
+
+        public static string normalize_padding(string value)
+        {
+            return value.ToLower();
+        }
+
+        public static string normalize_data_format(string value)
+        {
+            return value.ToLower();
+        }
     }
 }

src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs

@@ -13,14 +13,19 @@ namespace Tensorflow.Keras.Utils
             return tensors.Select(x => is_symbolic_tensor(x)).Count() == tensors.Length;
         }
 
+        public static bool? constant_value(Tensor pred)
+        {
+            return smart_module.smart_constant_value(pred);
+        }
+
         public static bool is_symbolic_tensor(Tensor tensor)
         {
             return true;
         }
 
-        public static object smart_cond(Tensor pred, 
-            Func<(Tensor, Tensor, Tensor)> true_fn = null,
-            Func<(Tensor, Tensor, Tensor)> false_fn = null, 
+        public static Tensor[] smart_cond<T>(Tensor pred, 
+            Func<T[]> true_fn = null,
+            Func<T[]> false_fn = null, 
             string name = null)
         {
             return smart_module.smart_cond(pred,

src/TensorFlowNET.Core/Keras/backend.cs

@@ -11,6 +11,22 @@ namespace Tensorflow.Keras
 
         }
 
+        public static Tensor placeholder(int[] shape = null, 
+            int ndim = -1, 
+            TF_DataType dtype = TF_DataType.DtInvalid, 
+            bool sparse = false, 
+            string name = null)
+        {
+            if(sparse)
+            {
+                throw new NotImplementedException("placeholder sparse is true");
+            }
+            else
+            {
+                return gen_array_ops.placeholder(dtype: dtype, shape: new TensorShape(shape), name: name);
+            }
+        }
+
         public static Graph get_graph()
         {
             return ops.get_default_graph();

src/TensorFlowNET.Core/Layers/Dense.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Operations.Activation;
+
+namespace Tensorflow.Layers
+{
+    public class Dense : Keras.Layers.Dense
+    {
+        public Dense(int units,
+            IActivation activation,
+            bool use_bias = true,
+            bool trainable = false,
+            IInitializer kernel_initializer = null) : base(units,
+                activation,
+                use_bias: use_bias,
+                trainable: trainable,
+                kernel_initializer: kernel_initializer)
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Layers/Layer.cs

@@ -1,11 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
-using Tensorflow.Keras.Engine;
 
 namespace Tensorflow.Layers
 {
-    public class Layer : Keras.Engine.Layer
+    public class Layer : Keras.Layers.Layer
     {
         protected Graph _graph;
         
@@ -52,14 +52,26 @@ namespace Tensorflow.Layers
 
             Python.with(scope_context_manager, scope2 => _current_scope = scope2);
             // Actually call layer
-            var outputs = base.__call__(inputs, training: training);
+            var outputs = base.__call__(new Tensor[] { inputs }, training: training);
 
             // Update global default collections.
-            //_add_elements_to_collection(updates, ops.GraphKeys.UPDATE_OPS);
+            _add_elements_to_collection(_updates.ToArray(), new string[] { ops.GraphKeys.UPDATE_OPS });
 
             return outputs;
         }
 
+        protected virtual void _add_elements_to_collection(Operation[] elements, string[] collection_list)
+        {
+            foreach(var name in collection_list)
+            {
+                var collection = ops.get_collection_ref(name) as List<object>;
+
+                foreach (var element in elements)
+                    if (!collection.Contains(element))
+                        collection.Add(element);
+            }
+        }
+
         protected virtual RefVariable add_weight(string name,
             int[] shape,
             TF_DataType dtype = TF_DataType.DtInvalid,

src/TensorFlowNET.Core/Operations/ControlFlows/CondContext.cs

@@ -10,22 +10,23 @@ namespace Tensorflow.Operations
     public class CondContext : ControlFlowContext
     {
         private string _name;
+
         /// <summary>
         /// The boolean tensor for the cond predicate
         /// </summary>
         private Tensor _pred;
-        /// <summary>
-        /// The predicate tensor in this branch
-        /// </summary>
-        private Tensor _pivot;
+        public Tensor pred => _pred;
+
         /// <summary>
         /// 0 or 1 representing this branch
         /// </summary>
         private int _branch;
+
         /// <summary>
         /// 
         /// </summary>
         private List<string> _values = new List<string>();
+
         private Dictionary<string, Tensor> _external_values = new Dictionary<string, Tensor>();
 
         /// <summary>
@@ -63,14 +64,23 @@ namespace Tensorflow.Operations
             }
         }
 
-        public (Tensor, Tensor, Tensor) BuildCondBranch(Func<(Tensor, Tensor, Tensor)> fn)
+        public (T[], Tensor[]) BuildCondBranch<T>(Func<T[]> fn)
         {
             // Add the subgraph defined by fn() to the graph.
             var pre_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION);
             var original_result = fn();
             var post_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION);
 
-            return original_result;
+            switch (original_result)
+            {
+                case Tensor[] results:
+                    return (original_result, results);
+                case float[] fv:
+                    var result = ops.convert_to_tensor(fv[0]);
+                    return (original_result, new Tensor[] { result });
+                default:
+                    return (original_result, new Tensor[0]);
+            }
         }
     }
 }

src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs

@@ -6,6 +6,11 @@ namespace Tensorflow.Operations
 {
     public abstract class ControlFlowContext : IPython, IControlFlowContext
     {
+        /// <summary>
+        /// The predicate tensor in this branch
+        /// </summary>
+        protected Tensor _pivot;
+
         protected Stack<IControlFlowContext> _context_stack;
         public ControlFlowContext()
         {
@@ -28,6 +33,29 @@ namespace Tensorflow.Operations
             graph._set_control_flow_context(this);
         }
 
+        public void AddOp(Operation op)
+        {
+            _AddOpInternal(op);
+        }
+
+        protected virtual void _AddOpInternal(Operation op)
+        {
+            if(op.inputs.Length == 0)
+            {
+                _RemoveExternalControlEdges(op);
+                op._add_control_input(_pivot.op);
+            }
+            else
+            {
+
+            }
+        }
+
+        protected virtual void _RemoveExternalControlEdges(Operation op)
+        {
+            var internal_control_inputs = op.control_inputs;
+        }
+
         public void Exit()
         {
             var graph = ops.get_default_graph();

src/TensorFlowNET.Core/Operations/ControlFlows/IControlFlowContext.cs

@@ -6,5 +6,6 @@ namespace Tensorflow
 {
     public interface IControlFlowContext
     {
+        void AddOp(Operation op);
     }
 }

src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Operations.Initializers
+{
+    public class RandomUniform : IInitializer
+    {
+        private int? seed;
+        private float minval;
+        private float maxval;
+        private TF_DataType dtype;
+
+        public RandomUniform()
+        {
+
+        }
+
+        public Tensor call(TensorShape shape, TF_DataType dtype = TF_DataType.DtInvalid)
+        {
+            return random_ops.random_uniform(shape, 
+                minval: minval, 
+                maxval: maxval, 
+                dtype: dtype, 
+                seed: seed);
+        }
+
+        public object get_config()
+        {
+            return new {
+                minval,
+                maxval,
+                seed,
+                dtype
+            };
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs

@@ -11,9 +11,9 @@ namespace Tensorflow.Operations.Initializers
         private int? seed;
         private TF_DataType dtype;
 
-        public TruncatedNormal(float mean = 0.0f, 
-            float stddev = 1.0f, 
-            int? seed = null, 
+        public TruncatedNormal(float mean = 0.0f,
+            float stddev = 1.0f,
+            int? seed = null,
             TF_DataType dtype = TF_DataType.TF_FLOAT)
         {
             this.mean = mean;
@@ -24,7 +24,7 @@ namespace Tensorflow.Operations.Initializers
 
         public Tensor call(TensorShape shape, TF_DataType dtype)
         {
-            throw new NotImplementedException("");
+            return random_ops.truncated_normal(shape, mean, stddev, dtype : dtype, seed: seed);
         }
 
         public object get_config()

src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Operations
+{
+    public class MaxPoolFunction : Python, IPoolFunction
+    {
+        public Tensor Apply(Tensor value,
+            int[] ksize,
+            int[] strides,
+            string padding,
+            string data_format = "NHWC",
+            string name = null)
+        {
+            return with(ops.name_scope(name, "MaxPool", new { value }), scope => {
+
+                value = ops.convert_to_tensor(value, name: "input");
+                return gen_nn_ops.max_pool(
+                    value,
+                    ksize: ksize,
+                    strides: strides,
+                    padding: padding,
+                    data_format: data_format,
+                    name: name);
+            });
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs

@@ -53,7 +53,23 @@ namespace Tensorflow.Operations
             return _op.outputs[0];
         }
 
-        public static (Tensor, Tensor, Tensor) _fused_batch_norm(Tensor x,
+        public static Tensor bias_add_grad(Tensor out_backprop,
+            string data_format = "NHWC",
+            string name = null)
+        {
+            if (data_format == null)
+                data_format = "NHWC";
+
+            var _op = _op_def_lib._apply_op_helper("BiasAddGrad", name: name, args: new
+            {
+                out_backprop,
+                data_format
+            });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor[] _fused_batch_norm(Tensor x,
                 Tensor scale,
                 Tensor offset,
                 Tensor mean,
@@ -75,7 +91,87 @@ namespace Tensorflow.Operations
                 is_training
             });
 
-            return (_op.outputs[0], _op.outputs[1], _op.outputs[2]);
+            return _op.outputs;
+        }
+
+        public static Tensor log_softmax(Tensor logits, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("LogSoftmax", name: name, args: new
+            {
+                logits
+            });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor max_pool(Tensor input,
+            int[] ksize,
+            int[] strides,
+            string padding,
+            string data_format = "NHWC",
+            string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("MaxPool", name: name, args: new
+            {
+                input,
+                ksize,
+                strides,
+                padding,
+                data_format,
+            });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor[] top_kv2(Tensor input, int k, bool sorted = true, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("TopKV2", name: name, args: new
+            {
+                input,
+                k,
+                sorted
+            });
+
+            return _op.outputs;
+        }
+
+        public static Tensor relu_grad(Tensor gradients, Tensor features, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("ReluGrad", name: name, args: new
+            {
+                gradients,
+                features
+            });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor softmax(Tensor logits, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Softmax", name: name, args: new
+            {
+                logits
+            });
+
+            return _op.outputs[0];
+        }
+
+        /// <summary>
+        /// Computes softmax cross entropy cost and gradients to backpropagate.
+        /// </summary>
+        /// <param name="features"></param>
+        /// <param name="labels"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static (Tensor, Tensor) softmax_cross_entropy_with_logits(Tensor features, Tensor labels, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("SoftmaxCrossEntropyWithLogits", name: name, args: new
+            {
+                features,
+                labels
+            });
+
+            return (_op.outputs[0], _op.outputs[1]);
         }
     }
 }

src/TensorFlowNET.Core/Operations/OpDefLibrary.cs

@@ -42,7 +42,7 @@ namespace Tensorflow
             var attrs = new Dictionary<string, object>();
             var inputs = new List<Tensor>();
             var input_types = new List<TF_DataType>();
-            dynamic values = null;
+            object values = null;
 
             return with(ops.name_scope(name), scope =>
             {
@@ -116,7 +116,7 @@ namespace Tensorflow
                         else if (default_type_attr_map.ContainsKey(input_arg.TypeAttr))
                             default_dtype = (DataType)default_type_attr_map[input_arg.TypeAttr];
 
-                        values = ops.internal_convert_to_tensor(values, 
+                        var value = ops.internal_convert_to_tensor(values, 
                             name: input_name, 
                             dtype: dtype.as_tf_dtype(),
                             as_ref: input_arg.IsRef,
@@ -125,7 +125,7 @@ namespace Tensorflow
                         //if (!String.IsNullOrEmpty(input_arg.TypeAttr))
                             //attrs[input_arg.TypeAttr] = values.dtype;
 
-                        values = new Tensor[] { values };
+                        values = new Tensor[] { value };
                     }
 
                     if (values is Tensor[] values2)

src/TensorFlowNET.Core/Operations/Operation.Control.cs

@@ -7,7 +7,7 @@ namespace Tensorflow
 {
     public partial class Operation
     {
-        private CondContext _control_flow_context;
+        private IControlFlowContext _control_flow_context;
 
         /// <summary>
         /// Add this op to its control flow context.
@@ -18,19 +18,30 @@ namespace Tensorflow
             {
 
             }
+
+            if (_control_flow_context != null)
+                _control_flow_context.AddOp(this);
+        }
+
+        public void _add_control_input(Operation op)
+        {
+            c_api.TF_AddControlInput(_handle, op);
         }
 
         public void _add_control_inputs(Operation[] ops)
         {
-            foreach(var op in ops)
-            {
-                c_api.TF_AddControlInput(graph, op);
-            }
+            foreach (var op in ops)
+                _add_control_input(op);
         }
 
-        public void _set_control_flow_context(CondContext ctx)
+        public void _set_control_flow_context(IControlFlowContext ctx)
         {
             _control_flow_context = ctx;
         }
+
+        public IControlFlowContext _get_control_flow_context()
+        {
+            return _control_flow_context;
+        }
     }
 }

src/TensorFlowNET.Core/Operations/Operation.Output.cs

@@ -1,4 +1,5 @@
-using System;
+//using Newtonsoft.Json;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.InteropServices;

src/TensorFlowNET.Core/Operations/Operation.cs

@@ -1,4 +1,5 @@
 using Google.Protobuf.Collections;
+//using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -102,8 +103,12 @@ namespace Tensorflow
                 }
             }
 
+            // Dict mapping op name to file and line information for op colocation
+            // context managers.
+            _control_flow_context = graph._get_control_flow_context();
+
             // This will be set by self.inputs.
-            if(op_def == null)
+            if (op_def == null)
                 op_def = g.GetOpDef(node_def.Op);
 
             var grouped_inputs = _reconstruct_sequence_inputs(op_def, inputs, node_def.Attr);
@@ -185,7 +190,10 @@ namespace Tensorflow
             if (oneof_value == "type")
                 return x.Type;
 
-            return x.GetType().GetProperty(oneof_value).GetValue(x);
+            object result = x.GetType().GetProperty(oneof_value).GetValue(x);
+            if (result is Google.Protobuf.ByteString byteString)
+                return byteString.ToStringUtf8();
+            return result;
         }
 
         public TF_AttrMetadata GetAttributeMetadata(string attr_name, Status s)

src/TensorFlowNET.Core/Operations/array_ops.py.cs

@@ -7,7 +7,8 @@ namespace Tensorflow
 {
     public class array_ops : Python
     {
-        public static Tensor placeholder_with_default<T>(T input, int[] shape, string name = null) => gen_array_ops.placeholder_with_default(input, shape, name);
+        public static Tensor placeholder_with_default<T>(T input, int[] shape, string name = null) 
+            => gen_array_ops.placeholder_with_default(input, shape, name);
 
         public static Tensor zeros(Shape shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null)
         {
@@ -41,20 +42,85 @@ namespace Tensorflow
             else
             {
                 tShape = constant_op._tensor_shape_tensor_conversion_function(shape.as_shape());
-                var c = constant_op.constant(0);
+                var c = constant_op.constant(0, dtype: dtype);
                 return gen_array_ops.fill(tShape, c, name: name);
             }
         }
 
-        public static Tensor expand_dims(Tensor input, int axis = -1, string name = null, int dim = -1) => expand_dims_v2(input, axis, name);
+        public static Tensor _autopacking_conversion_function(object[] v, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool as_ref = false)
+        {
+            var inferred_dtype = _get_dtype_from_nested_lists(v);
+            if (dtype == TF_DataType.DtInvalid)
+                dtype = inferred_dtype;
 
-        private static Tensor expand_dims_v2(Tensor input, int axis, string name = null) => gen_array_ops.expand_dims(input, axis, name);
+            return _autopacking_helper(v, dtype, name == null ? "packed" : name);
+        }
 
-        public static Tensor rank(Tensor input, string name = null)
+        private static TF_DataType _get_dtype_from_nested_lists(object[] list_or_tuple)
         {
-            return math_ops.rank_internal(input, name, optimize: true);
+            TF_DataType dtype = TF_DataType.DtInvalid;
+
+            foreach(var obj in list_or_tuple)
+            {
+                switch (obj)
+                {
+                    case Tensor t:
+                        dtype = t.dtype.as_base_dtype();
+                        break;
+                }
+
+                if (dtype != TF_DataType.DtInvalid)
+                    break;
+            }
+
+            return dtype;
         }
 
+        public static Tensor _autopacking_helper(object[] list_or_tuple, TF_DataType dtype, string name)
+        {
+            var must_pack = false;
+            var converted_elems = new List<object>();
+            return with(ops.name_scope(name), scope =>
+            {
+                foreach (var (i, elem) in enumerate(list_or_tuple))
+                {
+                    converted_elems.Add(elem);
+                    must_pack = true;
+                }
+
+                if(must_pack)
+                {
+                    var elems_as_tensors = new List<Tensor>();
+                    foreach (var (i, elem) in enumerate(converted_elems))
+                    {
+                        if (elem is Tensor tensor)
+                            elems_as_tensors.Add(tensor);
+                        else
+                        {
+                            var elem_tensor = constant_op.constant(elem, dtype: dtype, name: i.ToString());
+                            elems_as_tensors.Add(elem_tensor);
+                        }
+                    }
+
+                    return gen_array_ops.pack(elems_as_tensors.ToArray(), name: scope);
+                }
+                else
+                {
+                    // return converted_elems.ToArray();
+                    throw new NotImplementedException("_autopacking_helper.converted_elems");
+                }
+            });
+        }
+
+        public static Tensor expand_dims(Tensor input, int axis = -1, string name = null, int dim = -1) 
+            => expand_dims_v2(input, axis, name);
+
+        private static Tensor expand_dims_v2(Tensor input, int axis, string name = null) 
+            => gen_array_ops.expand_dims(input, axis, name);
+
+        public static Tensor rank(Tensor input, string name = null)
+            => math_ops.rank_internal(input, name, optimize: true);
+
         /// <summary>
         /// Creates a tensor with all elements set to 1.
         /// </summary>
@@ -66,10 +132,8 @@ namespace Tensorflow
         public static Tensor ones_like<T>(T tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true)
             => ones_like_impl(tensor, dtype, name, optimize);
 
-        public static Tensor reshape(Tensor tensor, Tensor shape, string name = null)
-        {
-            return gen_array_ops.reshape(tensor, shape, null);
-        }
+        public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
+            => gen_array_ops.reshape(tensor, shape, null);
 
         private static Tensor ones_like_impl<T>(T tensor, TF_DataType dtype, string name, bool optimize = true)
         {
@@ -97,6 +161,18 @@ namespace Tensorflow
             });
         }
 
+        public static Tensor ones(Tensor[] shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null)
+        {
+            dtype = dtype.as_base_dtype();
+            return with(ops.name_scope(name, "ones", new { shape }), scope =>
+            {
+                name = scope;
+                var shape1 = ops.convert_to_tensor(shape, dtype: TF_DataType.TF_INT32);
+                var output = gen_array_ops.fill(shape1, constant_op.constant(1, dtype: dtype), name: name);
+                return output;
+            });
+        }
+
         public static Tensor ones(int[] dims, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null)
         {
             dtype = dtype.as_base_dtype();
@@ -109,6 +185,44 @@ namespace Tensorflow
             });
         }
 
+        public static Tensor one_hot(Tensor indices, int depth, 
+            Tensor on_value = null,
+            Tensor off_value = null,
+            TF_DataType dtype = TF_DataType.DtInvalid,
+            int axis = -1,
+            string name = null)
+        {
+            return with(ops.name_scope(name, "one_hot", new { indices, depth, dtype }), scope =>
+            {
+                name = scope;
+                var on_exists = false;
+                var off_exists = false;
+                var on_dtype = TF_DataType.DtInvalid;
+                var off_dtype = TF_DataType.DtInvalid;
+
+                if (dtype == TF_DataType.DtInvalid)
+                    dtype = TF_DataType.TF_FLOAT;
+
+                if(!on_exists)
+                {
+                    on_value = ops.convert_to_tensor(1, dtype, name: "on_value");
+                    on_dtype = dtype;
+                }
+
+                if (!off_exists)
+                {
+                    off_value = ops.convert_to_tensor(0, dtype, name = "off_value");
+                    off_dtype = dtype;
+                }
+
+                return gen_array_ops.one_hot(indices, depth,
+                    on_value: on_value,
+                    off_value: off_value,
+                    axis: axis,
+                    name: name);
+            });
+        }
+
         public static Tensor where(Tensor condition, Tensor x = null, Tensor y = null, string name = null)
         {
             if( x == null && y == null)
@@ -136,14 +250,10 @@ namespace Tensorflow
         /// </param>
         /// <returns>A `Tensor` of type `out_type`.</returns>
         public static Tensor shape(Tensor input, string name = null, TF_DataType out_type = TF_DataType.TF_INT32)
-        {
-            return shape_internal(input, name, optimize: true, out_type: out_type);
-        }
+            => shape_internal(input, name, optimize: true, out_type: out_type);
 
         public static Tensor size(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32)
-        {
-            return size_internal(input, name, optimize: optimize, out_type: out_type);
-        }
+            => size_internal(input, name, optimize: optimize, out_type: out_type);
 
         private static Tensor shape_internal(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32)
         {
@@ -168,32 +278,21 @@ namespace Tensorflow
 
         private static Tensor size_internal(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32)
         {
-            return with(ops.name_scope(name, "Size", new Tensor[] { input }), scope =>
+            return with(ops.name_scope(name, "Size", new { input }), scope =>
             {
                 name = scope;
 
-                if (!tf.context.executing_eagerly())
+                var input_tensor = ops.convert_to_tensor(input);
+                var input_shape = tensor_util.to_shape(input_tensor.shape);
+                if (optimize)
                 {
-                    var input_tensor = ops.convert_to_tensor(input);
-                    var input_shape = tensor_util.to_shape(input_tensor.shape);
-                    if (optimize)
+                    if (input_shape.is_fully_defined())
                     {
-                        if (input_shape.is_fully_defined())
-                        {
-                            var nd = np.array(input_tensor.shape, out_type.as_numpy_datatype());
-                            return constant_op.constant(nd, name: name);
-                        }
+                        return constant_op.constant(input_shape.Size, dtype: out_type, name: name);
                     }
-
-                    return gen_array_ops.size(input, name: name, out_type: out_type);
-                }
-                else
-                {
-                    // result = gen_array_ops.shape();
-                    throw new NotImplementedException("array_ops.size_internal");
                 }
 
-                return null;
+                return gen_array_ops.size(input, name: name, out_type: out_type);
             });
         }
 
@@ -234,8 +333,46 @@ namespace Tensorflow
         /// <param name="name"></param>
         /// <returns></returns>
         public static Tensor stop_gradient(Tensor input, string name = null)
+            => gen_array_ops.stop_gradient(input,  name);
+
+        /// <summary>
+        /// Extracts a strided slice of a tensor (generalized python array indexing).
+        /// </summary>
+        /// <param name="input_"></param>
+        /// <param name="begin"></param>
+        /// <param name="end"></param>
+        /// <param name="strides"></param>
+        /// <param name="begin_mask"></param>
+        /// <param name="end_mask"></param>
+        /// <param name="ellipsis_mask"></param>
+        /// <param name="new_axis_mask"></param>
+        /// <param name="shrink_axis_mask"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor strided_slice(Tensor input_, Tensor begin, Tensor end, 
+            Tensor strides = null,
+            int begin_mask = 0,
+            int end_mask = 0,
+            int ellipsis_mask = 0,
+            int new_axis_mask = 0,
+            int shrink_axis_mask = 0,
+            string name = null)
         {
-            return gen_array_ops.stop_gradient(input,  name);
+            var op = gen_array_ops.strided_slice(
+                input: input_,
+                begin: begin,
+                end: end,
+                strides: strides,
+                begin_mask: begin_mask,
+                end_mask: end_mask,
+                ellipsis_mask: ellipsis_mask,
+                new_axis_mask: new_axis_mask,
+                shrink_axis_mask: shrink_axis_mask,
+                name: name);
+
+            string parent_name = name;
+
+            return op;
         }
 
         /// <summary>
@@ -256,14 +393,14 @@ namespace Tensorflow
         /// Contains the same data as `input`, but has one or more dimensions of
         /// size 1 removed.</returns>
         public static Tensor squeeze(Tensor input, int[] axis = null, string name = null, int[] squeeze_dims = null)
-        {
-            return gen_array_ops.squeeze(input, axis, name);
-        }
+            => gen_array_ops.squeeze(input, axis, name);
 
         public static Tensor identity(Tensor input, string name = null)
-        {
-            return gen_array_ops.identity(input, name);
-        }
+            => gen_array_ops.identity(input, name);
+
+        public static Tensor invert_permutation(Tensor x, string name = null)
+            => gen_array_ops.invert_permutation(x, name: name);
+
         /// <summary>
         /// Computes the shape of a broadcast given symbolic shapes.
         /// When shape_x and shape_y are Tensors representing shapes(i.e.the result of
@@ -279,27 +416,33 @@ namespace Tensorflow
         /// <param name="shape_y"> A rank 1 integer `Tensor`, representing the shape of y.</param>
         /// <returns> A rank 1 integer `Tensor` representing the broadcasted shape.</returns>
         public static Tensor broadcast_dynamic_shape(Tensor shape_x, Tensor shape_y)
-        {
-            return gen_array_ops.broadcast_args(shape_x, shape_y);
-        }
+            => gen_array_ops.broadcast_args(shape_x, shape_y);
 
         public static Tensor broadcast_static_shape(Tensor shape_x, Tensor shape_y)
-        {
-            return Framework.common_shapes.broadcast_shape(shape_x, shape_y);
-        }
+            => Framework.common_shapes.broadcast_shape(shape_x, shape_y);
 
         public static Tensor gather(Tensor @params, Tensor indices, string name = null, int axis = 0)
-        {
-            return gen_array_ops.gather_v2(@params, indices, axis, name: name);
-        }
+            => gen_array_ops.gather_v2(@params, indices, axis, name: name);
 
-        public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false)
+        public static Tensor transpose<T1, T2>(T1 a, T2 perm, string name = "transpose", bool conjugate = false)
         {
             return with(ops.name_scope(name, "transpose", new { a }), scope =>
             {
-                name = scope;
-                return gen_array_ops.transpose(a, perm, name);
+                return gen_array_ops.transpose(a, perm, name: scope);
             });
         }
+
+        public static Tensor slice<Tb, Ts>(Tensor input, Tb[] begin, Ts[] size, string name = null)
+            => gen_array_ops.slice(input, begin, size, name: name);
+
+        public static Tensor stack(object values, int axis = 0, string name = "stack")
+        {
+            if (axis == 0)
+                // If the input is a constant list, it can be converted to a constant op
+                return ops.convert_to_tensor(values, name: name);
+
+            throw new NotImplementedException("array_ops.stack");
+        }
+
     }
 }

src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using Tensorflow.Operations;
+using util = Tensorflow.control_flow_util;
 
 namespace Tensorflow
 {
@@ -137,9 +138,25 @@ namespace Tensorflow
                 return gen_array_ops.identity(data, name: name);
         }
 
-        public static (Tensor, Tensor) cond(Tensor pred,
-            Func<(Tensor, Tensor, Tensor)> true_fn = null,
-            Func<(Tensor, Tensor, Tensor)> false_fn = null,
+        /// <summary>
+        /// Forwards `data` to an output determined by `pred`.
+        /// </summary>
+        /// <param name="data"></param>
+        /// <param name="pred"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static (Tensor, Tensor) _SwitchRefOrTensor(Tensor data, Tensor pred, string name = "Switch")
+        {
+            data = ops.convert_to_tensor_or_indexed_slices(data, name: "data");
+
+            ops.colocate_with(data, ignore_existing: true);
+
+            return @switch(data, pred, name: name);
+        }
+
+        public static Tensor[] cond<T>(Tensor pred,
+            Func<T[]> true_fn = null,
+            Func<T[]> false_fn = null,
             bool strict = false,
             string name = null)
         {
@@ -158,20 +175,46 @@ namespace Tensorflow
                 // Build the graph for the true branch in a new context.
                 var context_t = new CondContext(pred, pivot_1, branch: 1);
                 context_t.Enter();
-                var res_t = context_t.BuildCondBranch(true_fn);
+                var (orig_res_t, res_t) = context_t.BuildCondBranch(true_fn);
                 context_t.Exit();
 
                 // Build the graph for the false branch in a new context.
                 var context_f = new CondContext(pred, pivot_2, branch: 0);
                 context_f.Enter();
-                var res_f = context_f.BuildCondBranch(false_fn);
+                var (orig_res_f, res_f) = context_f.BuildCondBranch(false_fn);
                 context_f.Exit();
 
-                var res_t_flat = new Tensor[] { res_t.Item1, res_t.Item2, res_t.Item3 };
-                var res_f_flat = new Tensor[] { res_f.Item1, res_f.Item2, res_f.Item3 };
+                var res_t_flat = res_t;
+                var res_f_flat = res_f;
+
+                var merges = zip(res_f_flat, res_t_flat)
+                    .Select(pair => merge(new Tensor[] { pair.Item1, pair.Item2 }))
+                    .ToArray();
+
+                merges = _convert_flows_to_tensorarrays(orig_res_t, merges);
+
+                ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_t);
+                ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_f);
 
+                return merges;
+            });
+        }
 
-                return (p_2, p_1);
+        public static Tensor[] _convert_flows_to_tensorarrays<T>(T[] tensors_or_tensorarrays, Tensor[] tensors_or_flows)
+        {
+            // zip(tensors_or_tensorarrays, tensors_or_flows).Select((ta, t_or_flow) => ta).ToArray();
+            return tensors_or_flows;
+        }
+
+        public static Tensor merge(Tensor[] inputs, string name = null)
+        {
+            return with(ops.name_scope(name, "Merge", inputs), scope =>
+            {
+                name = scope;
+                inputs = inputs.Select(inp =>
+                            ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref: true))
+                        .ToArray();
+                return gen_control_flow_ops.merge(inputs, name).Item1;
             });
         }
 
@@ -200,5 +243,18 @@ namespace Tensorflow
                 return gen_control_flow_ops.@switch(data, pred, name: name);
             });
         }
+
+        public static Tensor ZerosLikeOutsideLoop(Operation op, int index)
+        {
+            var val = op.outputs[index];
+            if (!util.IsSwitch(op))
+            {
+                if (val.dtype == TF_DataType.TF_RESOURCE)
+                    throw new NotImplementedException("ZerosLikeOutsideLoop");
+                return array_ops.zeros_like(val, optimize: false);
+            }
+
+            throw new NotImplementedException("ZerosLikeOutsideLoop");
+        }
     }
 }

src/TensorFlowNET.Core/Operations/control_flow_util.py.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Operations;
 
 namespace Tensorflow
 {
@@ -15,5 +16,22 @@ namespace Tensorflow
         {
             return op.type == "Exit" || op.type == "RefExit";
         }
+
+        /// <summary>
+        /// Return true if `op` is a Switch.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <returns></returns>
+        public static bool IsSwitch(Operation op)
+        {
+            return op.type == "Switch" || op.type == "RefSwitch";
+        }
+
+        public static IControlFlowContext GetOutputContext(Operation op)
+        {
+            var ctxt = op._get_control_flow_context();
+
+            return ctxt;
+        }
     }
 }

src/TensorFlowNET.Core/Operations/gen_array_ops.cs

@@ -27,16 +27,9 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
-        public static Tensor greater<Tx, Ty>(Tx x, Ty y, string name = null)
+        public static Tensor pack(Tensor[] values, int axis = 0, string name = null)
         {
-            var _op = _op_def_lib._apply_op_helper("Greater", name: name, args: new { x, y });
-
-            return _op.outputs[0];
-        }
-
-        public static Tensor less<Tx, Ty>(Tx x, Ty y, string name = null)
-        {
-            var _op = _op_def_lib._apply_op_helper("Less", name: name, args: new { x, y });
+            var _op = _op_def_lib._apply_op_helper("Pack", name: name, args: new { values, axis });
 
             return _op.outputs[0];
         }
@@ -68,6 +61,13 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        public static Tensor invert_permutation(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("InvertPermutation", name, new { x });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor log(Tensor x, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Log", name: name, args: new { x });
@@ -110,7 +110,13 @@ namespace Tensorflow
             return (_op.outputs[0], _op.outputs[1]);
         }
 
-        public static Tensor reshape(Tensor tensor, Tensor shape, string name = null)
+        public static Tensor reshape<T1, T2>(T1 tensor, T2 shape, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Reshape", name, new { tensor, shape });
+            return _op.outputs[0];
+        }
+
+        public static Tensor reshape(Tensor tensor, int[] shape, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Reshape", name, new { tensor, shape });
             return _op.outputs[0];
@@ -121,6 +127,17 @@ namespace Tensorflow
             throw new NotImplementedException("where");
         }
 
+        public static Tensor one_hot(Tensor indices, int depth,
+            Tensor on_value = null,
+            Tensor off_value = null,
+            TF_DataType dtype = TF_DataType.DtInvalid, 
+            int axis = -1,
+            string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("OneHot", name, new { indices, depth, on_value, off_value, axis });
+            return _op.outputs[0];
+        }
+
         /// <summary>
         /// A placeholder op that passes through `input` when its output is not fed.
         /// </summary>
@@ -140,6 +157,12 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        public static Tensor scatter_nd(Tensor indices, Tensor updates, Tensor[] shape, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("ScatterNd", name, new { indices, updates, shape });
+            return _op.outputs[0];
+        }
+
         public static Tensor shape(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Shape", name, new { input, out_type });
@@ -163,7 +186,7 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
-        public static Tensor transpose(Tensor x, int[] perm, string name = null)
+        public static Tensor transpose<T1, T2>(T1 x, T2 perm, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Transpose", name, new { x, perm });
             return _op.outputs[0];
@@ -174,12 +197,44 @@ namespace Tensorflow
             var _op = _op_def_lib._apply_op_helper("ZerosLike", name, new { x });
             return _op.outputs[0];
         }
+
         public static Tensor stop_gradient(Tensor x, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("StopGradient", name, args: new { input = x, name });
 
             return _op.outputs[0];
         }
+
+        public static Tensor strided_slice(Tensor input, Tensor begin, Tensor end, Tensor strides,
+            int begin_mask = 0,
+            int end_mask = 0,
+            int ellipsis_mask = 0,
+            int new_axis_mask = 0,
+            int shrink_axis_mask = 0,
+            string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("StridedSlice", name, new
+            {
+                input,
+                begin,
+                end,
+                strides,
+                begin_mask,
+                end_mask,
+                ellipsis_mask,
+                new_axis_mask,
+                shrink_axis_mask
+            });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor slice<Tb, Ts>(Tensor input, Tb[] begin, Ts[] size, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Slice", name, new { input, begin, size });
+            return _op.outputs[0];
+        }
+
         /// <summary>
         /// Removes dimensions of size 1 from the shape of a tensor.
         /// Given a tensor `input`, this operation returns a tensor of the same type with

src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs

@@ -4,7 +4,7 @@ using System.Text;
 
 namespace Tensorflow
 {
-    public class gen_control_flow_ops
+    public class gen_control_flow_ops : Python
     {
         public static OpDefLibrary _op_def_lib = new OpDefLibrary();
 
@@ -21,5 +21,12 @@ namespace Tensorflow
 
             return (_op.outputs[0], _op.outputs[1]);
         }
+
+        public static (Tensor, Tensor) merge(Tensor[] inputs, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Merge", name, new { inputs });
+
+            return (_op.outputs[0], _op.outputs[1]);
+        }
     }
 }

src/TensorFlowNET.Core/Operations/gen_math_ops.cs

@@ -9,6 +9,30 @@ namespace Tensorflow
     public static class gen_math_ops
     {
         public static OpDefLibrary _op_def_lib = new OpDefLibrary();
+
+        /// <summary>
+        /// Returns the index with the largest value across dimensions of a tensor.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <param name="dimension"></param>
+        /// <param name="output_type"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor arg_max(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null)
+            => _op_def_lib._apply_op_helper("ArgMax", name, args: new { input, dimension, output_type }).outputs[0];
+
+        /// <summary>
+        /// Returns the index with the smallest value across dimensions of a tensor.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <param name="dimension"></param>
+        /// <param name="output_type"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor arg_min(Tensor input, int dimension, TF_DataType output_type= TF_DataType.TF_INT64, string name= null)
+            =>_op_def_lib._apply_op_helper("ArgMin", name, args: new { input, dimension, output_type }).outputs[0];
+        
+
         /// <summary>
         /// Computes the mean of elements across dimensions of a tensor.
         /// Reduces `input` along the dimensions given in `axis`. Unless
        /// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
        /// `axis`. If `keep_dims` is true, the reduced dimensions are retained with length 1.
@@ -20,16 +44,30 @@ namespace Tensorflow
         /// <param name="keep_dims"> An optional `bool`. Defaults to `False`. If true, retain reduced dimensions with length 1.</param>
         /// <param name="name"> A name for the operation (optional).</param>
         /// <returns> A `Tensor`. Has the same type as `input`.</returns>
-        public static Tensor mean(Tensor input, Tensor axis, bool keep_dims= false, string name = null)
+        public static Tensor mean<T1, T2>(T1 input, T2 axis, bool keep_dims= false, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Mean", name, args: new { input, reduction_indices = axis, keep_dims = keep_dims });
 
             return _op.outputs[0];
         }
 
-        public static Tensor mean(Tensor input, int[] axis, bool keep_dims = false, string name = null)
+        public static Tensor prod<T1, T2>(T1 input, T2 axis, bool keep_dims = false, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Prod", name, args: new { input, reduction_indices = axis, keep_dims });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor acos(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Acos", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor asin(Tensor x, string name = null)
         {
-            var _op = _op_def_lib._apply_op_helper("Mean", name, args: new { input, reduction_indices = axis, keep_dims = keep_dims, name });
+            var _op = _op_def_lib._apply_op_helper("Asin", name, args: new { x });
 
             return _op.outputs[0];
         }
@@ -41,6 +79,83 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        public static Tensor atan(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Atan", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor ceil(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Ceil", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor cos(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Cos", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor cosh(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Cosh", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor floor(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Floor", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor _clip_by_value(Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("ClipByValue", name, args: new { t, clip_value_min, clip_value_max });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor greater<Tx, Ty>(Tx x, Ty y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Greater", name: name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor greater_equal<Tx, Ty>(Tx x, Ty y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("GreaterEqual", name: name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor less<Tx, Ty>(Tx x, Ty y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Less", name: name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor less_equal<Tx, Ty>(Tx x, Ty y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("LessEqual", name: name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor log1p(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Log1p", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor squared_difference(Tensor x, Tensor y, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("SquaredDifference", name, args: new { x, y, name });
@@ -128,6 +243,27 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        /// <summary>
+        /// Returns the truth value of (x == y) element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor equal(Tensor x, Tensor y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Equal", name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor atan2(Tensor y, Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Atan2", name, args: new { y, x });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor mul(Tensor x, Tensor y, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Mul", name, args: new { x, y });
@@ -142,6 +278,13 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        public static Tensor reciprocal(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Reciprocal", name, args: new { x });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor floor_mod(Tensor x, Tensor y, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("FloorMod", name, args: new { x, y });
@@ -186,13 +329,34 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
-        public static Tensor _max(Tensor input, int[] axis, bool keep_dims=false, string name = null)
+        public static Tensor minimum<T1, T2>(T1 x, T2 y, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Minimum", name, args: new { x, y });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor _abs(Tensor x, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Abs", name, new { x });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor _max<Tx, Ty>(Tx input, Ty axis, bool keep_dims=false, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Max", name, new { input, reduction_indices = axis, keep_dims });
 
             return _op.outputs[0];
         }
 
+        public static Tensor _min<Tx, Ty>(Tx input, Ty axis, bool keep_dims = false, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Min", name, new { input, reduction_indices = axis, keep_dims });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor pow<Tx, Ty>(Tx x, Ty y, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Pow", name, args: new { x, y });
@@ -200,7 +364,14 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
-        public static Tensor sum(Tensor input, Tensor axis = null, bool keep_dims = false, string name = null)
+        public static Tensor _sum(Tensor input, Tensor axis = null, bool keep_dims = false, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Sum", name, args: new { input, reduction_indices = axis, keep_dims });
+
+            return _op.outputs[0];
+        }
+
+        public static Tensor _sum(Tensor input, int axis, bool keep_dims = false, string name = null)
         {
             var _op = _op_def_lib._apply_op_helper("Sum", name, args: new { input, reduction_indices = axis, keep_dims });
 

src/TensorFlowNET.Core/Operations/math_ops.py.cs → src/TensorFlowNET.Core/Operations/math_ops.cs

@@ -2,12 +2,40 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Framework;
 
 namespace Tensorflow
 {
+    /// <summary>
+    /// python\ops\math_ops.py
+    /// </summary>
     public class math_ops : Python
     {
-        public static Tensor add(Tensor x, Tensor y, string name = null) => gen_math_ops.add(x, y, name);
+        public static Tensor abs(Tensor x, string name = null)
+        {
+            return with(ops.name_scope(name, "Abs", new { x }), scope =>
+            {
+                x = ops.convert_to_tensor(x, name: "x");
+                if (x.dtype.is_complex())
+                    throw new NotImplementedException("math_ops.abs for dtype.is_complex");
+                    //return gen_math_ops.complex_abs(x, Tout: x.dtype.real_dtype, name: name);
+                return gen_math_ops._abs(x, name: name);
+            });
+        }
+
+        public static Tensor add(Tensor x, Tensor y, string name = null) 
+            => gen_math_ops.add(x, y, name);
+
+        public static Tensor add(Tensor x, string name = null)
+        {
+            return with(ops.name_scope(name, "Abs", new { x }), scope =>
+            {
+                name = scope;
+                x = ops.convert_to_tensor(x, name: "x");
+
+                return gen_math_ops._abs(x, name: name);
+            });
+        }
 
         public static Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null)
         {
@@ -17,6 +45,7 @@ namespace Tensorflow
 
             return with(ops.name_scope(name, "Cast", new { x }), scope =>
             {
+                name = scope;
                 x = ops.convert_to_tensor(x, name: "x");
                 if (x.dtype.as_base_dtype() != base_type)
                     x = gen_math_ops.cast(x, base_type, name: name);
@@ -36,12 +65,44 @@ namespace Tensorflow
         /// dimensions.Must be in the range `[-rank(input_tensor), rank(input_tensor))`.</param>
         /// <param name="keepdims"> If true, retains reduced dimensions with length 1.</param>
         /// <param name="name"> A name for the operation (optional).</param>
-        public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null)
+        public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null, int? reduction_indices = null)
         {
             var r = _ReductionDims(input_tensor, axis);
-            var m = gen_math_ops.mean(input_tensor, (int[]) r, keepdims, name);
-            return _may_reduce_to_scalar(keepdims,axis, m);
+            if (axis == null)
+            {
+                var m = gen_math_ops.mean(input_tensor, r, keepdims, name);
+                return _may_reduce_to_scalar(keepdims, axis, m);
+            }
+            else
+            {
+                var m = gen_math_ops.mean(input_tensor, axis, keepdims, name);
+                return _may_reduce_to_scalar(keepdims, axis, m);
+            }
+        }
+
+        /// <summary>
+        /// Computes the product of elements across dimensions of a tensor.
+        /// </summary>
+        /// <param name="input_tensor"></param>
+        /// <param name="axis"></param>
+        /// <param name="keepdims"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor reduce_prod(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null)
+        {
+            var r = _ReductionDims(input_tensor, axis);
+            if (axis == null)
+            {
+                var m = gen_math_ops.prod(input_tensor, r, keepdims, name);
+                return _may_reduce_to_scalar(keepdims, axis, m);
+            }
+            else
+            {
+                var m = gen_math_ops.prod(input_tensor, axis, keepdims, name);
+                return _may_reduce_to_scalar(keepdims, axis, m);
+            }
         }
+        
         /// <summary>
         /// Returns (x - y)(x - y) element-wise.
         /// </summary>
@@ -60,6 +121,11 @@ namespace Tensorflow
             return gen_math_ops.square(x, name);
         }
 
+        public static Tensor subtract<Tx, Ty>(Tx x, Ty y, string name = null)
+        {
+            return gen_math_ops.sub(x, y, name);
+        }
+
         public static Tensor log(Tensor x, string name = null)
         {
             return gen_math_ops.log(x, name);
@@ -87,6 +153,16 @@ namespace Tensorflow
             return gen_data_flow_ops.dynamic_stitch(a1, a2);
         }
 
+        /// <summary>
+        /// Computes the reciprocal of x element-wise.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor reciprocal(Tensor x, string name = null)
+            => gen_math_ops.reciprocal(x, name: name);
+        
+
         /// <summary>
         /// Computes log(sum(exp(elements across dimensions of a tensor))).
         /// Reduces `input_tensor` along the dimensions given in `axis`.
@@ -129,7 +205,10 @@ namespace Tensorflow
 
         public static Tensor reduce_max(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null)
         {
-            return _may_reduce_to_scalar(keepdims, axis, gen_math_ops._max(input_tensor, (int[])_ReductionDims(input_tensor, axis), keepdims, name));
+            var r = _ReductionDims(input_tensor, axis);
+            var max = (axis != null) ? gen_math_ops._max(input_tensor, axis, keepdims, name) :
+                gen_math_ops._max(input_tensor, r, keepdims, name);
+            return _may_reduce_to_scalar(keepdims, axis, max);
         }
 
         /// <summary>
@@ -160,20 +239,29 @@ namespace Tensorflow
             throw new NotImplementedException();
         }
 
-        public static Tensor reduce_sum(Tensor input_tensor, Tensor axis = null, bool keepdims = false)
+        public static Tensor reduce_sum(Tensor input_tensor, Tensor axis = null, bool keepdims = false, string name = null)
         {
             var r = _ReductionDims(input_tensor, axis);
-            var m = gen_math_ops.sum(input_tensor, r);
-            return _may_reduce_to_scalar(keepdims, m);
+            var m = gen_math_ops._sum(input_tensor, r, keep_dims: keepdims, name: name);
+            return _may_reduce_to_scalar(keepdims, axis, m);
         }
 
-        private static Tensor _may_reduce_to_scalar(bool keepdims, Tensor output)
+        public static Tensor reduce_sum(Tensor input_tensor, int axis, bool keepdims = false, string name = null)
         {
-            output.shape = new long[0];
+            var m = gen_math_ops._sum(input_tensor, axis, keep_dims: keepdims, name: name);
+            return _may_reduce_to_scalar(keepdims, new int[] { axis }, m);
+        }
+
+        private static Tensor _may_reduce_to_scalar(bool keepdims, Tensor axis, Tensor output)
+        {
+            if (!common_shapes.has_fully_defined_shape(output) &&
+                !keepdims &&
+                axis == null)
+                output.shape = new long[0];
             return output;
         }
 
-        private static Tensor _may_reduce_to_scalar(bool keepdims, int[] axos, Tensor output)
+        private static Tensor _may_reduce_to_scalar(bool keepdims, int[] axis, Tensor output)
         {
             output.shape = new long[0];
             return output;
@@ -191,19 +279,20 @@ namespace Tensorflow
                 return range(0, rank, 1);
             }
         }
-        
-        private static object _ReductionDims(Tensor x, int[] axis)
+
+        private static Tensor _ReductionDims(Tensor x, int[] axis)
         {
             if (axis != null)
             {
-                return axis;
+                // should return axis. or check before.
+                return null;
             }
             else
             {
-                var rank = array_ops.rank(x);
+                var rank = common_shapes.rank(x);
                 if (rank != null)
                 {
-                   return constant_op.constant(np.arange(rank), TF_DataType.TF_INT32);
+                   return constant_op.constant(np.arange(rank.Value), TF_DataType.TF_INT32);
                 }
                 return range(0, rank, 1);
             }
@@ -221,7 +310,7 @@ namespace Tensorflow
             if (delta == null)
                 delta = 1;
 
-            return with(ops.name_scope(name, "Range", new object[] { start, limit, delta }), scope =>
+            return with(ops.name_scope(name, "Range", new { start, limit, delta }), scope =>
             {
                 name = scope;
                 var start1 = ops.convert_to_tensor(start, name: "start");
@@ -298,5 +387,20 @@ namespace Tensorflow
                 return x;
             });
         }
+
+        public static Tensor truediv(Tensor x, Tensor y, string name = null)
+            => _truediv_python3(x, y, name);
+
+        public static Tensor _truediv_python3(Tensor x, Tensor y, string name = null)
+        {
+            return with(ops.name_scope(name, "truediv", new { x, y }), scope =>
+            {
+                name = scope;
+                var x_dtype = x.dtype.as_base_dtype();
+                var y_dtype = y.dtype.as_base_dtype();
+
+                return gen_math_ops.real_div(x, y, name: name);
+            });
+        }
     }
 }

src/TensorFlowNET.Core/Operations/nn_impl.py.cs

@@ -46,7 +46,7 @@ namespace Tensorflow
             });
         }
 
-        public static (Tensor, Tensor, Tensor) fused_batch_norm(Tensor x, 
+        public static Tensor[] fused_batch_norm(Tensor x, 
             RefVariable scale,
             RefVariable offset,
             Tensor mean,

src/TensorFlowNET.Core/Operations/nn_ops.cs

@@ -41,5 +41,55 @@ namespace Tensorflow
                 return gen_nn_ops.bias_add(value, bias_tensor, data_format: data_format, name: name);
             });
         }
+
+        public static Tensor log_softmax(Tensor logits, int axis = -1, string name = null)
+        {
+            return _softmax(logits, gen_nn_ops.log_softmax, axis, name);
+        }
+
+        public static Tensor _softmax(Tensor logits, Func<Tensor, string, Tensor> compute_op, int dim = -1, string name = null)
+        {
+            logits = ops.convert_to_tensor(logits);
+
+            var shape = logits.shape;
+            bool is_last_dim = dim == -1 || dim == shape.Length - 1;
+            if (is_last_dim)
+                return compute_op(logits, name);
+
+            throw new NotImplementedException("_softmax helper");
+        }
+
+        public static Tensor softmax_cross_entropy_with_logits_v2_helper(Tensor labels,
+            Tensor logits,
+            int axis = -1,
+            string name = null)
+        {
+            return Python.with(ops.name_scope(name, "softmax_cross_entropy_with_logits", new { }), scope =>
+            {
+                var precise_logits = logits;
+                var input_rank = array_ops.rank(precise_logits);
+                var shape = logits.getShape();
+
+                if (axis != -1)
+                    throw new NotImplementedException("softmax_cross_entropy_with_logits_v2_helper axis != -1");
+
+                var input_shape = array_ops.shape(precise_logits);
+
+                // Do the actual op computation.
+                // The second output tensor contains the gradients.  We use it in
+                // _CrossEntropyGrad() in nn_grad but not here.
+
+                var (cost, unused_backprop) = gen_nn_ops.softmax_cross_entropy_with_logits(precise_logits, labels, name: name);
+
+                // The output cost shape should be the input minus axis.
+                var output_shape = array_ops.slice(input_shape, 
+                    new int[] { 0 },
+                    new Tensor[] { math_ops.subtract(input_rank, 1) });
+
+                cost = array_ops.reshape(cost, output_shape);
+
+                return cost;
+            });
+        }
     }
 }

src/TensorFlowNET.Core/Python.cs

@@ -2,6 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Linq;
 using System.Text;
 
 namespace Tensorflow
@@ -16,6 +17,11 @@ namespace Tensorflow
             Console.WriteLine(obj.ToString());
         }
 
+        protected IEnumerable<int> range(int end)
+        {
+            return Enumerable.Range(0, end);
+        }
+
         public static T New<T>(object args) where T : IPyClass
         {
             var instance = Activator.CreateInstance<T>();
@@ -118,14 +124,6 @@ namespace Tensorflow
             {
                 object obj = propertyDescriptor.GetValue(dyn);
                 string name = propertyDescriptor.Name;
-                // avoid .net keyword
-                switch (name)
-                {
-                    case "_ref_":
-                        name = "ref";
-                        break;
-                }
-
                 dictionary.Add(name, obj);
             }
             return dictionary;

src/TensorFlowNET.Core/Sessions/BaseSession.cs

@@ -186,9 +186,10 @@ namespace Tensorflow
             var result = new NDArray[fetch_list.Length];
 
             for (int i = 0; i < fetch_list.Length; i++)
-            {
                 result[i] = fetchValue(output_values[i]);
-            }
+
+            for (int i = 0; i < feed_dict.Length; i++)
+                feed_dict[i].Value.Dispose();
 
             return result;
         }
@@ -222,6 +223,12 @@ namespace Tensorflow
                         ints[i] = *(int*)(offset + (int)(tensor.itemsize * i));
                     nd = np.array(ints).reshape(ndims);
                     break;
+                case TF_DataType.TF_INT64:
+                    var longs = new long[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        longs[i] = *(long*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(longs).reshape(ndims);
+                    break;
                 case TF_DataType.TF_FLOAT:
                     var floats = new float[tensor.size];
                     for (ulong i = 0; i < tensor.size; i++)

src/TensorFlowNET.Core/Sessions/_ElementFetchMapper.cs

@@ -10,7 +10,6 @@ namespace Tensorflow
     /// </summary>
     public class _ElementFetchMapper : _FetchMapper
     {
-        private List<object> _unique_fetches = new List<object>();
         private Func<List<object>, object> _contraction_fn;
 
         public _ElementFetchMapper(object[] fetches, Func<List<object>, object> contraction_fn)
@@ -32,7 +31,7 @@ namespace Tensorflow
         /// </summary>
         /// <param name="values"></param>
         /// <returns></returns>
-        public NDArray build_results(List<object> values)
+        public override NDArray build_results(List<object> values)
         {
             NDArray result = null;
 
@@ -44,6 +43,24 @@ namespace Tensorflow
                     case NDArray value:
                         result = value;
                         break;
+                    case short value:
+                        result = value;
+                        break;
+                    case int value:
+                        result = value;
+                        break;
+                    case long value:
+                        result = value;
+                        break;
+                    case float value:
+                        result = value;
+                        break;
+                    case double value:
+                        result = value;
+                        break;
+                    case string value:
+                        result = value;
+                        break;
                     default:
                         break;
                 }
@@ -51,10 +68,5 @@ namespace Tensorflow
 
             return result;
         }
-
-        public List<object> unique_fetches()
-        {
-            return _unique_fetches;
-        }
     }
 }

src/TensorFlowNET.Core/Sessions/_FetchHandler.cs

@@ -10,7 +10,7 @@ namespace Tensorflow
     /// </summary>
     public class _FetchHandler
     {
-        private _ElementFetchMapper _fetch_mapper;
+        private _FetchMapper _fetch_mapper;
         private List<Tensor> _fetches = new List<Tensor>();
         private List<bool> _ops = new List<bool>();
         private List<Tensor> _final_fetches = new List<Tensor>();
@@ -18,7 +18,7 @@ namespace Tensorflow
 
         public _FetchHandler(Graph graph, object fetches, Dictionary<object, object> feeds = null, Action feed_handles = null)
         {
-            _fetch_mapper = new _FetchMapper().for_fetch(fetches);
+            _fetch_mapper = _FetchMapper.for_fetch(fetches);
             foreach(var fetch in _fetch_mapper.unique_fetches())
             {
                 switch (fetch)
@@ -58,7 +58,31 @@ namespace Tensorflow
                 {
                     var value = tensor_values[j];
                     j += 1;
-                    full_values.Add(value);
+                    if (value.ndim == 0)
+                    {
+                        switch (value.dtype.Name)
+                        {
+                            case "Int32":
+                                full_values.Add(value.Data<int>(0));
+                                break;
+                            case "Int64":
+                                full_values.Add(value.Data<long>(0));
+                                break;
+                            case "Single":
+                                full_values.Add(value.Data<float>(0));
+                                break;
+                            case "Double":
+                                full_values.Add(value.Data<double>(0));
+                                break;
+                            case "String":
+                                full_values.Add(value.Data<string>(0));
+                                break;
+                        }
+                    }
+                    else
+                    {
+                        full_values.Add(value[np.arange(0, value.shape[0])]);
+                    }
                 }
                 i += 1;
             }

src/TensorFlowNET.Core/Sessions/_FetchMapper.cs

@@ -1,4 +1,5 @@
-using System;
+using NumSharp.Core;
+using System;
 using System.Collections.Generic;
 using System.Text;
 
@@ -6,14 +7,26 @@ namespace Tensorflow
 {
     public class _FetchMapper
     {
-        public _ElementFetchMapper for_fetch(object fetch)
+        protected List<object> _unique_fetches = new List<object>();
+
+        public static _FetchMapper for_fetch(object fetch)
         {
-            var fetches = new object[] { fetch };
+            var fetches = fetch.GetType().IsArray ? (object[])fetch : new object[] { fetch };
+
+            if (fetch.GetType().IsArray)
+                return new _ListFetchMapper(fetches);
+            else
+                return new _ElementFetchMapper(fetches, (List<object> fetched_vals) => fetched_vals[0]);
+        }
 
-            return new _ElementFetchMapper(fetches, (List<object> fetched_vals) =>
-            {
-                return fetched_vals[0];
-            });
+        public virtual NDArray build_results(List<object> values)
+        {
+            return values.ToArray();
+        }
+
+        public virtual List<object> unique_fetches()
+        {
+            return _unique_fetches;
         }
     }
 }

src/TensorFlowNET.Core/Sessions/_ListFetchMapper.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow
+{
+    public class _ListFetchMapper : _FetchMapper
+    {
+        private _FetchMapper[] _mappers;
+
+        public _ListFetchMapper(object[] fetches)
+        {
+            _mappers = fetches.Select(fetch => _FetchMapper.for_fetch(fetch)).ToArray();
+            _unique_fetches.AddRange(fetches);
+        }
+    }
+}

src/TensorFlowNET.Core/TensorFlowNET.Core.csproj

@@ -4,23 +4,26 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <AssemblyName>TensorFlow.NET</AssemblyName>
     <RootNamespace>Tensorflow</RootNamespace>
-    <Version>0.4.2</Version>
+    <Version>0.6.0</Version>
     <Authors>Haiping Chen</Authors>
     <Company>SciSharp STACK</Company>
-    <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+    <GeneratePackageOnBuild>false</GeneratePackageOnBuild>
     <Copyright>Apache 2.0</Copyright>
     <RepositoryUrl>https://github.com/SciSharp/TensorFlow.NET</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
     <PackageProjectUrl>https://github.com/SciSharp</PackageProjectUrl>
     <PackageIconUrl>https://avatars3.githubusercontent.com/u/44989469?s=200&amp;v=4</PackageIconUrl>
-    <PackageTags>TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET</PackageTags>
+    <PackageTags>TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET, C#</PackageTags>
     <Description>Google's TensorFlow binding in .NET Standard.
 Docs: https://tensorflownet.readthedocs.io</Description>
-    <AssemblyVersion>0.4.2.0</AssemblyVersion>
-    <PackageReleaseNotes>Added ConfigProto to control CPU and GPU resource.
-Fixed import name scope issue.</PackageReleaseNotes>
+    <AssemblyVersion>0.6.0.0</AssemblyVersion>
+    <PackageReleaseNotes>Changes since v0.5:
+Added K-means Clustering.
+Added Nearest Neighbor.
+Added a lot of APIs to build neural networks model.
+Bug fix.</PackageReleaseNotes>
     <LangVersion>7.2</LangVersion>
-    <FileVersion>0.4.2.0</FileVersion>
+    <FileVersion>0.6.0.0</FileVersion>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
@@ -44,15 +47,17 @@ Fixed import name scope issue.</PackageReleaseNotes>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.7.0" />
-    <PackageReference Include="NumSharp" Version="0.7.4" />
+    <PackageReference Include="NumSharp" Version="0.8.2" />
   </ItemGroup>
 
   <ItemGroup>
-    <Content CopyToOutputDirectory="PreserveNewest" Include="./runtimes/win-x64/native/tensorflow.dll" Link="tensorflow.dll" Pack="true" PackagePath="runtimes/win-x64/native/tensorflow.dll" />
+    <Content CopyToOutputDirectory="PreserveNewest" Include="../../tensorflowlib/runtimes/win-x64/native/tensorflow.dll" Link="tensorflow.dll" Pack="true" PackagePath="../../tensorflowlib/runtimes/win-x64/native/tensorflow.dll" />
+    <Content CopyToOutputDirectory="PreserveNewest" Include="../../tensorflowlib/runtimes/linux-x64/native/libtensorflow.so" Link="libtensorflow.so" Pack="true" PackagePath="../../tensorflowlib/runtimes/linux-x64/native/libtensorflow.so" />
+    <Content CopyToOutputDirectory="PreserveNewest" Include="../../tensorflowlib/runtimes/linux-x64/native/libtensorflow_framework.so" Link="libtensorflow_framework.so" Pack="true" PackagePath="../../tensorflowlib/runtimes/linux-x64/native/libtensorflow_framework.so" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\..\NumSharp\src\NumSharp.Core\NumSharp.Core.csproj" />
+    <Folder Include="Keras\Initializers\" />
   </ItemGroup>
 
 </Project>

src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs

@@ -111,7 +111,7 @@ namespace Tensorflow
             // Free the original buffer and set flag
             Deallocator deallocator = (IntPtr values, IntPtr len, ref bool closure) =>
             {
-                Marshal.FreeHGlobal(dotHandle);
+                Marshal.FreeHGlobal(values);
                 closure = true;
             };
 

src/TensorFlowNET.Core/Tensors/Tensor.Operators.cs

@@ -14,6 +14,7 @@ namespace Tensorflow
         public static Tensor operator -(Tensor x, Tensor y) => BinaryOpWrapper("sub", x, y);
         public static Tensor operator -(Tensor x, int y) => BinaryOpWrapper("sub", x, y);
         public static Tensor operator -(Tensor x, double y) => BinaryOpWrapper("sub", x, y);
+        public static Tensor operator -(float x, Tensor y) => BinaryOpWrapper("Sub", x, y);
 
         public static Tensor operator *(float x, Tensor y) => BinaryOpWrapper("mul", x, y);
         public static Tensor operator *(double x, Tensor y) => BinaryOpWrapper("mul", x, y);
@@ -26,12 +27,12 @@ namespace Tensorflow
 
         public static Tensor operator %(Tensor x, Tensor y) => BinaryOpWrapper("mod", x, y);
 
-        public static Tensor operator >(Tensor x, int y) => gen_array_ops.greater(x, y);
-        public static Tensor operator >(Tensor x, float y) => gen_array_ops.greater(x, y);
-        public static Tensor operator >(Tensor x, double y) => gen_array_ops.greater(x, y);
-        public static Tensor operator <(Tensor x, int y) => gen_array_ops.less(x, y);
-        public static Tensor operator <(Tensor x, float y) => gen_array_ops.less(x, y);
-        public static Tensor operator <(Tensor x, double y) => gen_array_ops.less(x, y);
+        public static Tensor operator >(Tensor x, int y) => gen_math_ops.greater(x, y);
+        public static Tensor operator >(Tensor x, float y) => gen_math_ops.greater(x, y);
+        public static Tensor operator >(Tensor x, double y) => gen_math_ops.greater(x, y);
+        public static Tensor operator <(Tensor x, int y) => gen_math_ops.less(x, y);
+        public static Tensor operator <(Tensor x, float y) => gen_math_ops.less(x, y);
+        public static Tensor operator <(Tensor x, double y) => gen_math_ops.less(x, y);
 
         private static Tensor BinaryOpWrapper<Tx, Ty>(string name, Tx x, Ty y)
         {
@@ -48,7 +49,7 @@ namespace Tensorflow
                 var x1 = ops.convert_to_tensor(x, dtype: dtype, name: "x");
                 var y1 = ops.convert_to_tensor(y, dtype: dtype, name: "y");
 
-                switch (name)
+                switch (name.ToLower())
                 {
                     case "add":
                         result = gen_math_ops.add(x1, y1, name: scope);

src/TensorFlowNET.Core/Tensors/Tensor.cs

@@ -1,4 +1,5 @@
-using NumSharp.Core;
+//using Newtonsoft.Json;
+using NumSharp.Core;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -43,6 +44,8 @@ namespace Tensorflow
         public IntPtr buffer => _handle == IntPtr.Zero ? IntPtr.Zero : c_api.TF_TensorData(_handle);
         public int num_consumers(TF_Output oper_out) => _handle == IntPtr.Zero ? 0 : c_api.TF_OperationOutputNumConsumers(oper_out);
 
+        private TF_Output? _tf_output;
+
         public long[] shape
         {
             get
@@ -74,7 +77,8 @@ namespace Tensorflow
 
         public int[] _shape_tuple()
         {
-            return null;
+            if (shape == null) return null;
+            return shape.Select(x => (int)x).ToArray();
         }
 
         public TensorShape getShape()
@@ -122,7 +126,10 @@ namespace Tensorflow
 
         public TF_Output _as_tf_output()
         {
-            return new TF_Output(op, value_index);
+            if(!_tf_output.HasValue)
+                _tf_output = new TF_Output(op, value_index);
+
+            return _tf_output.Value;
         }
 
         public T[] Data<T>()
@@ -161,7 +168,12 @@ namespace Tensorflow
         /// <param name="feed_dict">A dictionary that maps `Tensor` objects to feed values.</param>
         /// <param name="session">The `Session` to be used to evaluate this tensor.</param>
         /// <returns></returns>
-        public NDArray eval(FeedItem[] feed_dict = null, Session session = null)
+        public NDArray eval(params FeedItem[] feed_dict)
+        {
+            return ops._eval_using_default_session(this, feed_dict, graph);
+        }
+
+        public NDArray eval(Session session, FeedItem[] feed_dict = null)
         {
             return ops._eval_using_default_session(this, feed_dict, graph, session);
         }
@@ -186,6 +198,61 @@ namespace Tensorflow
             }
         }
 
+        public Tensor this[int slice_spec]
+        {
+            get
+            {
+                var slice_spec_s = new int[] { slice_spec };
+                var begin = new List<int>();
+                var end = new List<int>();
+                var strides = new List<int>();
+
+                var index = 0;
+                var (new_axis_mask, shrink_axis_mask) = (0, 0);
+                var (begin_mask, end_mask) = (0, 0);
+                var ellipsis_mask = 0;
+
+                foreach(var s in slice_spec_s)
+                {
+                    {
+                        begin.Add(s);
+                        end.Add(s + 1);
+                        strides.Add(1);
+                        shrink_axis_mask |= (1 << index);
+                    }
+                    
+                    index += 1;
+                }
+
+                return with(ops.name_scope(null, "strided_slice", new { begin, end, strides }), scope =>
+                {
+                    string name = scope;
+                    if(begin != null)
+                    {
+                        var (packed_begin, packed_end, packed_strides) =
+                            (array_ops.stack(begin.ToArray()),
+                            array_ops.stack(end.ToArray()),
+                            array_ops.stack(strides.ToArray()));
+
+                        return gen_array_ops.strided_slice(
+                            this,
+                            packed_begin,
+                            packed_end,
+                            packed_strides,
+                            begin_mask: begin_mask,
+                            end_mask: end_mask,
+                            shrink_axis_mask: shrink_axis_mask,
+                            new_axis_mask: new_axis_mask,
+                            ellipsis_mask: ellipsis_mask,
+                            name: name);
+                    }
+
+                    throw new NotImplementedException("");
+                });
+            }
+            
+        }
+
         public override string ToString()
         {
             if(NDims == 0)

src/TensorFlowNET.Core/Tensors/dtypes.cs

@@ -31,9 +31,15 @@ namespace Tensorflow
 
             switch (type.Name)
             {
+                case "Boolean":
+                    dtype = TF_DataType.TF_BOOL;
+                    break;
                 case "Int32":
                     dtype = TF_DataType.TF_INT32;
                     break;
+                case "Int64":
+                    dtype = TF_DataType.TF_INT64;
+                    break;
                 case "Single":
                     dtype = TF_DataType.TF_FLOAT;
                     break;

src/TensorFlowNET.Core/Tensors/tensor_util.cs

@@ -38,7 +38,8 @@ namespace Tensorflow
             {
                 return MakeNdarray(tensor.op.get_attr("value") as TensorProto);
             }
-            throw new NotImplementedException("_ConstantValue");   
+
+            return null;
         }
 
         public static NDArray MakeNdarray(TensorProto tensor)
@@ -50,6 +51,15 @@ namespace Tensorflow
             if (tensor.TensorContent.Length > 0)
                 return np.frombuffer(tensor.TensorContent.ToByteArray(), tensor_dtype)
                     .reshape(shape);
+            else if (tensor.Dtype == DataType.DtHalf || tensor.Dtype == DataType.DtBfloat16)
+                ;
+            else if (tensor.Dtype == DataType.DtFloat)
+                ;
+            else if (new DataType[] { DataType.DtInt32, DataType.DtUint8 }.Contains(tensor.Dtype))
+                if (tensor.IntVal.Count == 1)
+                    return np.repeat(np.array(tensor.IntVal[0]), Convert.ToInt32(num_elements))
+                        .reshape(shape);
+
             throw new NotImplementedException("MakeNdarray");
         }
 
@@ -101,6 +111,9 @@ namespace Tensorflow
                         case int intVal:
                             nparray = intVal;
                             break;
+                        case long intVal:
+                            nparray = intVal;
+                            break;
                         case int[] intVals:
                             nparray = np.array(intVals);
                             break;
@@ -216,11 +229,15 @@ namespace Tensorflow
             switch (nparray.dtype.Name)
             {
                 case "Bool":
+                case "Boolean":
                     tensor_proto.BoolVal.AddRange(proto_values.Data<bool>());
                     break;
                 case "Int32":
                     tensor_proto.IntVal.AddRange(proto_values.Data<int>());
                     break;
+                case "Int64":
+                    tensor_proto.Int64Val.AddRange(proto_values.Data<long>());
+                    break;
                 case "Single":
                     tensor_proto.FloatVal.AddRange(proto_values.Data<float>());
                     break;
@@ -286,7 +303,7 @@ namespace Tensorflow
                     default:
                         throw new NotImplementedException("as_shape Not Implemented");
                 }
-                dim.Name = $"dim_{i}";
+                // dim.Name = $"dim_{i}";
 
                 shape.Dim.Add(dim);
             }
@@ -317,7 +334,7 @@ namespace Tensorflow
             {
                 var dim = new TensorShapeProto.Types.Dim();
                 dim.Size = tshape.Dimensions[i];
-                dim.Name = $"dim_{i}";
+                //dim.Name = $"dim_{i}";
 
                 shape.Dim.Add(dim);
             }

src/TensorFlowNET.Core/Train/AdamOptimizer.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Train
+{
+    /// <summary>
+    /// Optimizer that implements the Adam algorithm.
+    /// http://arxiv.org/abs/1412.6980
+    /// </summary>
+    public class AdamOptimizer : Optimizer
+    {
+        private float _beta1;
+        private float _beta2;
+        private float _epsilon;
+
+        public AdamOptimizer(float learning_rate, float beta1 = 0.9f, float beta2 = 0.999f, float epsilon = 1e-8f, bool use_locking = false, string name = "Adam")
+            : base(learning_rate, use_locking, name)
+        {
+            _beta1 = beta1;
+            _beta2 = beta2;
+            _epsilon = epsilon;
+        }
+    }
+}

src/TensorFlowNET.Core/Train/GradientDescentOptimizer.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace Tensorflow
+namespace Tensorflow.Train
 {
     public class GradientDescentOptimizer : Optimizer
     {

src/TensorFlowNET.Core/Train/Optimizer.cs

@@ -34,6 +34,7 @@ namespace Tensorflow
 
             Name = name;
             _use_locking = use_locking;
+            LearningRate = learning_rate;
             // Dictionary of slots.
             _slots = new Dictionary<string, object>();
             _non_slot_dict = new Dictionary<string, object>();
@@ -49,6 +50,7 @@ namespace Tensorflow
         /// was not `None`, that operation also increments `global_step`.
         /// </returns>
         public Operation minimize(Tensor loss, 
+            RefVariable global_step = null,
             GateGradientType gate_gradients = GateGradientType.GATE_OP,
             bool colocate_gradients_with_ops = false)
         {

src/TensorFlowNET.Core/Train/Saving/Saver.cs

@@ -251,7 +251,7 @@ namespace Tensorflow
         {
             return export_meta_graph(
                 filename: filename,
-                graph_def: ops.get_default_graph()._as_graph_def(add_shapes: true),
+                graph_def: ops.get_default_graph().as_graph_def(add_shapes: true),
                 saver_def: _saver_def,
                 collection_list: collection_list,
                 as_text: as_text,

src/TensorFlowNET.Core/Train/tf.optimizers.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
+using Tensorflow.Train;
 
 namespace Tensorflow
 {
@@ -11,9 +12,12 @@ namespace Tensorflow
         {
             public static Optimizer GradientDescentOptimizer(float learning_rate) => new GradientDescentOptimizer(learning_rate);
 
+            public static Optimizer AdamOptimizer(float learning_rate) => new AdamOptimizer(learning_rate);
+
             public static Saver Saver() => new Saver();
 
-            public static string write_graph(Graph graph, string logdir, string name, bool as_text = true) => graph_io.write_graph(graph, logdir, name, as_text);
+            public static string write_graph(Graph graph, string logdir, string name, bool as_text = true) 
+                => graph_io.write_graph(graph, logdir, name, as_text);
 
             public static Saver import_meta_graph(string meta_graph_or_file,
                 bool clear_devices = false,

src/TensorFlowNET.Core/Variables/RefVariable.Operators.cs

@@ -13,7 +13,8 @@ namespace Tensorflow
         public static Tensor operator -(RefVariable x, int y) => op_helper("sub", x, y);
         public static Tensor operator -(RefVariable x, float y) => op_helper("sub", x, y);
         public static Tensor operator -(RefVariable x, double y) => op_helper("sub", x, y);
-        
+        public static Tensor operator -(RefVariable x, Tensor y) => op_helper("sub", x, y);
+
         private static Tensor op_helper<T>(string default_name, RefVariable x, T y)
         {
             var tensor1 = x.value();

src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs

@@ -52,7 +52,7 @@ namespace Tensorflow
             bool use_locking = true,
             string name = null)
         {
-            var _op = _op_def_lib._apply_op_helper("Assign", name: name, args: new { _ref_ = tensor, value, validate_shape, use_locking });
+            var _op = _op_def_lib._apply_op_helper("Assign", name: name, args: new { @ref = tensor, value, validate_shape, use_locking });
 
             var _result = _op.outputs;
             var _inputs_flat = _op.inputs;
@@ -66,5 +66,15 @@ namespace Tensorflow
 
             return _result[0];
         }
+
+        public static Tensor assign_sub(RefVariable @ref,
+            Tensor value,
+            bool use_locking = false,
+            string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("AssignSub", name: name, args: new { @ref, value, use_locking });
+
+            return _op.outputs[0];
+        }
     }
 }

src/TensorFlowNET.Core/Variables/state_ops.cs

@@ -24,5 +24,13 @@ namespace Tensorflow
                 name: name, 
                 container: container, 
                 shared_name: shared_name);
+
+        public static Tensor assign_sub(RefVariable @ref,
+            Tensor value,
+            bool use_locking = false,
+            string name = null) => gen_state_ops.assign_sub(@ref,
+                value,
+                use_locking: use_locking,
+                name: name);
     }
 }

src/TensorFlowNET.Core/Variables/variable_scope.py.cs

@@ -156,6 +156,7 @@ namespace Tensorflow
             {
                 return new RefVariable(initial_value, 
                     trainable: trainable.Value,
+                    validate_shape: validate_shape,
                     name: name,
                     dtype: dtype);
             }

src/TensorFlowNET.Core/Variables/variables.py.cs

@@ -47,11 +47,11 @@ namespace Tensorflow
         /// special tokens filters by prefix.
         /// </param>
         /// <returns>A list of `Variable` objects.</returns>
-        public static List<RefVariable> global_variables(string scope = "")
+        public static List<RefVariable> global_variables(string scope = null)
         {
             var result = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope);
 
-            return result as List<RefVariable>;
+            return result == null ? new List<RefVariable>() : result as List<RefVariable>;
         }
 
         /// <summary>
@@ -62,7 +62,10 @@ namespace Tensorflow
         /// <returns>An Op that run the initializers of all the specified variables.</returns>
         public static Operation variables_initializer(RefVariable[] var_list, string name = "init")
         {
-            return control_flow_ops.group(var_list.Select(x => x.initializer).ToArray(), name);
+            if (var_list.Length > 0)
+                return control_flow_ops.group(var_list.Select(x => x.initializer).ToArray(), name);
+            else
+                return gen_control_flow_ops.no_op(name: name);
         }
     }
 }

src/TensorFlowNET.Core/ops.GraphKeys.cs

@@ -47,6 +47,10 @@ namespace Tensorflow
 
             // Used to store v2 summary names.
             public static string _SUMMARY_COLLECTION = "_SUMMARY_V2";
+
+            // Key for control flow context.
+            public static string COND_CONTEXT = "cond_context";
+            public static string WHILE_CONTEXT = "while_context";
         }
     }
 }

src/TensorFlowNET.Core/ops.py.cs

@@ -9,6 +9,7 @@ using Google.Protobuf;
 using System.Linq;
 using NumSharp.Core;
 using System.ComponentModel;
+using Tensorflow.Gradients;
 
 namespace Tensorflow
 {
@@ -40,7 +41,7 @@ namespace Tensorflow
         /// list contains the values in the order under which they were
         /// collected.
         /// </returns>
-        public static object get_collection(string key, string scope = "")
+        public static object get_collection(string key, string scope = null)
         {
             return get_default_graph().get_collection(key, scope);
         }
@@ -345,43 +346,6 @@ namespace Tensorflow
             session.run(operation, feed_dict);
         }
 
-        public static Func<Operation, Tensor, Tensor[]> get_gradient_function(Operation op)
-        {
-            if (op.inputs == null) return null;
-
-            return (oper, out_grads) =>
-            {
-                // Console.WriteLine($"get_gradient_function: {oper.type} '{oper.name}'");
-
-                switch (oper.type)
-                {
-                    case "Add":
-                        var add = math_grad._AddGrad(oper, out_grads);
-                        return new Tensor[] { add.Item1, add.Item2 };
-                    case "Identity":
-                        var id = math_grad._IdGrad(oper, out_grads);
-                        return new Tensor[] { id };
-                    case "Mul":
-                        var mul = math_grad._MulGrad(oper, out_grads);
-                        return new Tensor[] { mul.Item1, mul.Item2 };
-                    case "Sum":
-                        var sum = math_grad._SumGrad(oper, out_grads);
-                        return new Tensor[] { sum.Item1, sum.Item2 };
-                    case "Sub":
-                        var sub = math_grad._SubGrad(oper, out_grads);
-                        return new Tensor[] { sub.Item1, sub.Item2 };
-                    case "Pow":
-                        var pow = math_grad._PowGrad(oper, out_grads);
-                        return new Tensor[] { pow.Item1, pow.Item2 };
-                    case "RealDiv":
-                        var realdiv = math_grad._RealDivGrad(oper, out_grads);
-                        return new Tensor[] { realdiv.Item1, realdiv.Item2 };
-                    default:
-                        throw new NotImplementedException($"get_gradient_function {oper.type}");
-                }
-            };
-        }
-
         public static Tensor[] convert_n_to_tensor_or_indexed_slices(Tensor[] values, TF_DataType dtype = TF_DataType.DtInvalid, string name = null)
         {
             return internal_convert_n_to_tensor_or_indexed_slices(values, dtype: dtype, name: name);
@@ -417,13 +381,13 @@ namespace Tensorflow
             return ret.ToArray();
         }
 
-        public static Tensor[] internal_convert_n_to_tensor<T>(T[] values, TF_DataType dtype = TF_DataType.DtInvalid, 
+        public static Tensor[] internal_convert_n_to_tensor(object values, TF_DataType dtype = TF_DataType.DtInvalid, 
             string name = null, TF_DataType preferred_dtype = TF_DataType.DtInvalid, 
             bool as_ref = false)
         {
             var ret = new List<Tensor>();
 
-            foreach((int i, T value) in Python.enumerate(values))
+            foreach((int i, object value) in enumerate(values as object[]))
             {
                 string n = string.IsNullOrEmpty(name) ? "" : $"{name}_{i}";
                 ret.Add(internal_convert_to_tensor(value, dtype: dtype, name: n, as_ref: as_ref, preferred_dtype: preferred_dtype));
@@ -434,7 +398,8 @@ namespace Tensorflow
 
         public static Tensor internal_convert_to_tensor(object value, TF_DataType dtype = TF_DataType.DtInvalid,
             string name = null, TF_DataType preferred_dtype = TF_DataType.DtInvalid,
-            bool as_ref = false)
+            bool as_ref = false,
+            string scope = null)
         {
             if (dtype == TF_DataType.DtInvalid)
                 dtype = preferred_dtype;
@@ -445,24 +410,14 @@ namespace Tensorflow
                     return constant_op.constant(nd, dtype: dtype, name: name);
                 case Tensor tensor:
                     return tensor;
-                case string str:
-                    return constant_op.constant(str, dtype: dtype, name: name);
-                case string[] strArray:
-                    return constant_op.constant(strArray, dtype: dtype, name: name);
-                case int intVal:
-                    return constant_op.constant(intVal, dtype: dtype, name: name);
-                case int[] intArray:
-                    return constant_op.constant(intArray, dtype: dtype, name: name);
-                case float floatVal:
-                    return constant_op.constant(floatVal, dtype: dtype, name: name);
-                case float[] floatArray:
-                    return constant_op.constant(floatArray, dtype: dtype, name: name);
-                case double doubleVal:
-                    return constant_op.constant(doubleVal, dtype: dtype, name: name);
+                case Tensor[] tensors:
+                    return array_ops._autopacking_helper(tensors, dtype, name);
                 case RefVariable varVal:
                     return varVal._TensorConversionFunction(as_ref: as_ref);
+                case object[] objects:
+                    return array_ops._autopacking_conversion_function(objects, dtype: dtype, name: name);
                 default:
-                    throw new NotImplementedException($"internal_convert_to_tensor: Can't convert {value.GetType().Name} to Tensor");
+                    return constant_op.constant(value, dtype: dtype, name: name);
             }
         }
 

src/TensorFlowNET.Core/tf.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 using Tensorflow.Eager;
+using static Tensorflow.ops;
 
 namespace Tensorflow
 {
@@ -21,11 +22,13 @@ namespace Tensorflow
 
         public static RefVariable Variable<T>(T data, 
             bool trainable = true,
+            bool validate_shape = true,
             string name = null, 
             TF_DataType dtype = TF_DataType.DtInvalid)
         {
             return Tensorflow.variable_scope.default_variable_creator(data, 
                 trainable: trainable,
+                validate_shape: validate_shape,
                 name: name,
                 dtype: TF_DataType.DtInvalid);
         }

tensorflowlib/README.md

@@ -1,3 +1,5 @@
+TensorFlow.NET pack all required libraries in architecture-specific assemblies folders per NuGet standard.
+
 Here are some pre-built TensorFlow binaries you can use for each platform:
 
 - Linux
@@ -6,7 +8,18 @@ Here are some pre-built TensorFlow binaries you can use for each platform:
 - Mac: https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.13.1.tar.gz
 - Windows: https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-1.13.1.zip
 
+### Run in Linux
+
+`Install-Package TensorFlow.NET`
+
+Download Linux pre-built library and unzip `libtensorflow.so` and `libtensorflow_framework.so` into current running directory.
+
+### Run in Mac OS
+
+### Build from source for Windows
+
 https://www.tensorflow.org/install/source_windows
+
 pacman -S git patch unzip
 
 1. Build static library