diff --git a/.gitignore b/.gitignore index ad96cbd7..e4aba715 100644 --- a/.gitignore +++ b/.gitignore @@ -328,15 +328,8 @@ ASALocalRun/ # MFractors (Xamarin productivity tool) working folder .mfractor/ -/tensorflowlib/win7-x64/native/libtensorflow.dll -/tensorflowlib/osx/native/libtensorflow_framework.dylib -/tensorflowlib/osx/native/libtensorflow.dylib -/tensorflowlib/linux/native/libtensorflow_framework.so -/tensorflowlib/linux/native/libtensorflow.so -/src/TensorFlowNET.Core/tensorflow.dll /docs/build -src/TensorFlowNET.Native/libtensorflow.dll src/TensorFlowNET.Native/bazel-* -/src/TensorFlowNET.Native/libtensorflow.lib src/TensorFlowNET.Native/c_api.h /.vscode +test/TensorFlowNET.Examples/mnist diff --git a/README.md b/README.md index 7a861d04..da0bf7f6 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ Read the docs & book [The Definitive Guide to Tensorflow.NET](https://tensorflow * [Basic Operations](test/TensorFlowNET.Examples/BasicOperations.cs) * [Image Recognition](test/TensorFlowNET.Examples/ImageRecognition.cs) * [Linear Regression](test/TensorFlowNET.Examples/LinearRegression.cs) +* [Logistic Regression](test/TensorFlowNET.Examples/LogisticRegression.cs) +* [Nearest Neighbor](test/TensorFlowNET.Examples/NearestNeighbor.cs) * [Text Classification](test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs) * [CNN Text Classification](test/TensorFlowNET.Examples/CnnTextClassification.cs) * [Naive Bayes Classification](test/TensorFlowNET.Examples/NaiveBayesClassifier.cs) diff --git a/TensorFlow.NET.sln b/TensorFlow.NET.sln index 7470442b..e50bb267 100644 --- a/TensorFlow.NET.sln +++ b/TensorFlow.NET.sln @@ -11,8 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Core", "src\T EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Visualization", "src\TensorFlowNET.Visualization\TensorFlowNET.Visualization.csproj", "{0254BFF9-453C-4FE0-9609-3644559A79CE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NumSharp.Core", "..\NumSharp\src\NumSharp.Core\NumSharp.Core.csproj", "{3EEAFB06-BEF0-4261-BAAB-630EABD25290}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -35,10 +33,6 @@ Global {0254BFF9-453C-4FE0-9609-3644559A79CE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.Build.0 = Release|Any CPU - {3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docs/source/Graph.md b/docs/source/Graph.md index f6edbfc6..7bc473f2 100644 --- a/docs/source/Graph.md +++ b/docs/source/Graph.md @@ -21,3 +21,61 @@ A typical graph is looks like below: ![image](../assets/graph_vis_animation.gif) + + +### Save Model + +Saving the model means saving all the values of the parameters and the graph. + +```python +saver = tf.train.Saver() +saver.save(sess,'./tensorflowModel.ckpt') +``` + +After saving the model there will be four files: + +* tensorflowModel.ckpt.meta: +* tensorflowModel.ckpt.data-00000-of-00001: +* tensorflowModel.ckpt.index +* checkpoint + +We also created a protocol buffer file .pbtxt. It is human readable if you want to convert it to binary: `as_text: false`. + +* tensorflowModel.pbtxt: + +This holds a network of nodes, each representing one operation, connected to each other as inputs and outputs. + + + +### Freezing the Graph + +##### *Why we need it?* + +When we need to keep all the values of the variables and the Graph structure in a single file we have to freeze the graph. + +```csharp +from tensorflow.python.tools import freeze_graph + +freeze_graph.freeze_graph(input_graph = 'logistic_regression/tensorflowModel.pbtxt', + input_saver = "", + input_binary = False, + input_checkpoint = 'logistic_regression/tensorflowModel.ckpt', + output_node_names = "Softmax", + restore_op_name = "save/restore_all", + filename_tensor_name = "save/Const:0", + output_graph = 'frozentensorflowModel.pb', + clear_devices = True, + initializer_nodes = "") + +``` + +### Optimizing for Inference + +To Reduce the amount of computation needed when the network is used only for inferences we can remove some parts of a graph that are only needed for training. + + + +### Restoring the Model + + + diff --git a/docs/source/LinearRegression.md b/docs/source/LinearRegression.md index 27f777fb..d92f115f 100644 --- a/docs/source/LinearRegression.md +++ b/docs/source/LinearRegression.md @@ -8,6 +8,8 @@ Consider the case of a single variable of interest y and a single predictor vari We have some data $D=\{x{\tiny i},y{\tiny i}\}$ and we assume a simple linear model of this dataset with Gaussian noise: +线性回归是一种线性建模方法，这种方法用来描述自变量与一个或多个因变量的之间的关系。在只有一个因变量y和一个自变量的情况下。自变量还有以下几种叫法：协变量，输入，特征；因变量通常被叫做响应变量，输出，输出结果。 +假如我们有数据$D=\{x{\tiny i},y{\tiny i}\}$，并且假设这个数据集是满足高斯分布的线性模型： ```csharp // Prepare training Data var train_X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f, 7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f); @@ -18,6 +20,8 @@ var n_samples = train_X.shape[0]; Based on the given data points, we try to plot a line that models the points the best. The red line can be modelled based on the linear equation: $y = wx + b$. The motive of the linear regression algorithm is to find the best values for $w$ and $b$. Before moving on to the algorithm, le's have a look at two important concepts you must know to better understand linear regression. +按照上图根据数据描述的数据点，在这些数据点之间画出一条线，这条线能达到最好模拟点的分布的效果。红色的线能够通过下面呢线性等式来描述：$y = wx + b$。线性回归算法的目标就是找到这条线对应的最好的参数$w$和$b$。在介绍线性回归算法之前，我们先看两个重要的概念，这两个概念有助于你理解线性回归算法。 + ### Cost Function The cost function helps us to figure out the best possible values for $w$ and $b$ which would provide the best fit line for the data points. Since we want the best values for $w$ and $b$, we convert this search problem into a minimization problem where we would like to minimize the error between the predicted value and the actual value. @@ -65,4 +69,4 @@ When we visualize the graph in TensorBoard: ![linear-regression](_static/linear-regression-tensor-board.png) -The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LinearRegression.cs). \ No newline at end of file +The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LinearRegression.cs). diff --git a/docs/source/LogisticRegression.md b/docs/source/LogisticRegression.md new file mode 100644 index 00000000..00aa2f05 --- /dev/null +++ b/docs/source/LogisticRegression.md @@ -0,0 +1,7 @@ +# Chapter. Logistic Regression + +### What is logistic regression? + + + +The full example is [here](https://github.com/SciSharp/TensorFlow.NET/blob/master/test/TensorFlowNET.Examples/LogisticRegression.cs). \ No newline at end of file diff --git a/docs/source/NearestNeighbor.md b/docs/source/NearestNeighbor.md new file mode 100644 index 00000000..fa846e0c --- /dev/null +++ b/docs/source/NearestNeighbor.md @@ -0,0 +1,3 @@ +# Chapter. Nearest Neighbor + +The nearest neighbour algorithm was one of the first algorithms used to solve the travelling salesman problem. In it, the salesman starts at a random city and repeatedly visits the nearest city until all have been visited. It quickly yields a short tour, but usually not the optimal one. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index b41c621f..e26f8378 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -26,4 +26,6 @@ Welcome to TensorFlow.NET's documentation! Train EagerMode LinearRegression + LogisticRegression + NearestNeighbor ImageRecognition \ No newline at end of file diff --git a/src/TensorFlowNET.Core/APIs/keras.layers.cs b/src/TensorFlowNET.Core/APIs/keras.layers.cs new file mode 100644 index 00000000..00fe7ee1 --- /dev/null +++ b/src/TensorFlowNET.Core/APIs/keras.layers.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Keras; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers; + +namespace Tensorflow +{ + public static partial class keras + { + public static class layers + { + public static Embedding Embedding(int input_dim, int output_dim, + IInitializer embeddings_initializer = null, + bool mask_zero = false) => new Embedding(input_dim, output_dim, + embeddings_initializer, + mask_zero); + + public static Tensor[] Input(int[] batch_shape = null, + TF_DataType dtype = TF_DataType.DtInvalid, + string name = null, + bool sparse = false, + Tensor tensor = null) + { + var batch_size = batch_shape[0]; + var shape = batch_shape.Skip(1).ToArray(); + + var input_layer = new InputLayer( + input_shape: shape, + batch_size: batch_size, + name: name, + dtype: dtype, + sparse: sparse, + input_tensor: tensor); + + var outputs = input_layer.inbound_nodes[0].output_tensors; + + return outputs; + } + } + } +} diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs index c63fe7fe..a0fb3c72 100644 --- a/src/TensorFlowNET.Core/APIs/tf.array.cs +++ b/src/TensorFlowNET.Core/APIs/tf.array.cs @@ -28,7 +28,17 @@ namespace Tensorflow /// /// /// - public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false) + public static Tensor transpose(T1 a, T2 perm, string name = "transpose", bool conjugate = false) => array_ops.transpose(a, perm, name, conjugate); + + public static Tensor squeeze(Tensor input, int[] axis = null, string name = null, int squeeze_dims = -1) + => gen_array_ops.squeeze(input, axis, name); + + public static Tensor one_hot(Tensor indices, int depth, + Tensor on_value = null, + Tensor off_value = null, + TF_DataType dtype = TF_DataType.DtInvalid, + int axis = -1, + string name = null) => array_ops.one_hot(indices, depth, dtype: dtype, axis: axis, name: name); } } diff --git a/src/TensorFlowNET.Core/APIs/tf.control.cs b/src/TensorFlowNET.Core/APIs/tf.control.cs new file mode 100644 index 00000000..1b6bc3b8 --- /dev/null +++ b/src/TensorFlowNET.Core/APIs/tf.control.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow +{ + public static partial class tf + { + public static _ControlDependenciesController control_dependencies(Operation[] control_inputs) + => ops.control_dependencies(control_inputs); + } +} diff --git a/src/TensorFlowNET.Core/APIs/tf.init.cs b/src/TensorFlowNET.Core/APIs/tf.init.cs index fb5421c3..11ae5fe4 100644 --- a/src/TensorFlowNET.Core/APIs/tf.init.cs +++ b/src/TensorFlowNET.Core/APIs/tf.init.cs @@ -10,7 +10,8 @@ namespace Tensorflow public static IInitializer zeros_initializer => new Zeros(); public static IInitializer ones_initializer => new Ones(); public static IInitializer glorot_uniform_initializer => new GlorotUniform(); - + public static IInitializer uniform_initializer => new RandomUniform(); + public static variable_scope variable_scope(string name, string default_name = null, object values = null, @@ -27,5 +28,13 @@ namespace Tensorflow default_name, values, auxiliary_name_scope); + + public static IInitializer truncated_normal_initializer(float mean = 0.0f, + float stddev = 1.0f, + int? seed = null, + TF_DataType dtype = TF_DataType.DtInvalid) => new TruncatedNormal(mean: mean, + stddev: stddev, + seed: seed, + dtype: dtype); } } diff --git a/src/TensorFlowNET.Core/APIs/tf.layers.cs b/src/TensorFlowNET.Core/APIs/tf.layers.cs index dc883a75..faf0d089 100644 --- a/src/TensorFlowNET.Core/APIs/tf.layers.cs +++ b/src/TensorFlowNET.Core/APIs/tf.layers.cs @@ -100,6 +100,52 @@ namespace Tensorflow return layer.apply(inputs, training: training); } + + /// + /// Max pooling layer for 2D inputs (e.g. images). + /// + /// The tensor over which to pool. Must have rank 4. + /// + /// + /// + /// + /// + /// + public static Tensor max_pooling2d(Tensor inputs, + int[] pool_size, + int[] strides, + string padding = "valid", + string data_format = "channels_last", + string name = null) + { + var layer = new MaxPooling2D(pool_size: pool_size, + strides: strides, + padding: padding, + data_format: data_format, + name: name); + + return layer.apply(inputs); + } + + public static Tensor dense(Tensor inputs, + int units, + IActivation activation = null, + bool use_bias = true, + IInitializer kernel_initializer = null, + IInitializer bias_initializer = null, + bool trainable = true, + string name = null, + bool? reuse = null) + { + if (bias_initializer == null) + bias_initializer = tf.zeros_initializer; + + var layer = new Dense(units, activation, + use_bias: use_bias, + kernel_initializer: kernel_initializer); + + return layer.apply(inputs); + } } } } diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 9226ce63..24f09056 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -6,21 +6,235 @@ namespace Tensorflow { public static partial class tf { - public static Tensor add(Tensor a, Tensor b) => gen_math_ops.add(a, b); + public static Tensor abs(Tensor x, string name = null) + => math_ops.abs(x, name); - public static Tensor sub(Tensor a, Tensor b) => gen_math_ops.sub(a, b); + /// + /// Computes acos of x element-wise. + /// + /// + /// + /// + public static Tensor acos(Tensor x, string name = null) + => gen_math_ops.acos(x, name); + + /// + /// Computes asin of x element-wise. + /// + /// + /// + /// + public static Tensor asin(Tensor x, string name = null) + => gen_math_ops.asin(x, name); + + public static Tensor add(Tensor a, Tensor b) + => gen_math_ops.add(a, b); + + /// + /// Computes atan of x element-wise. + /// + /// + /// + /// + public static Tensor atan(Tensor x, string name = null) + => gen_math_ops.atan(x, name); + + public static Tensor arg_max(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) + => gen_math_ops.arg_max(input, dimension, output_type: output_type, name: name); + + public static Tensor arg_min(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) + => gen_math_ops.arg_min(input, dimension, output_type: output_type, name: name); + + /// + /// Returns element-wise smallest integer not less than x. + /// + /// + /// + /// + public static Tensor ceil(Tensor x, string name = null) + => gen_math_ops.ceil(x, name); + + /// + /// Computes cos of x element-wise. + /// + /// + /// + /// + public static Tensor cos(Tensor x, string name = null) + => gen_math_ops.cos(x, name); + + /// + /// Computes hyperbolic cosine of x element-wise. + /// + /// + /// + /// + public static Tensor cosh(Tensor x, string name = null) + => gen_math_ops.cosh(x, name); + + /// + /// Returns element-wise largest integer not greater than x. + /// + /// + /// + /// + public static Tensor floor(Tensor x, string name = null) + => gen_math_ops.floor(x, name); + + /// + /// Returns the truth value of (x > y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor greater(Tx x, Ty y, string name = null) + => gen_math_ops.greater(x, y, name); + + /// + /// Returns the truth value of (x >= y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor greater_equal(Tx x, Ty y, string name = null) + => gen_math_ops.greater_equal(x, y, name); + + /// + /// Returns the truth value of (x < y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor less(Tx x, Ty y, string name = null) + => gen_math_ops.less(x, y, name); - public static Tensor sqrt(Tensor a, string name = null) => gen_math_ops.sqrt(a, name); + /// + /// Returns the truth value of (x <= y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor less_equal(Tx x, Ty y, string name = null) + => gen_math_ops.less_equal(x, y, name); + + /// + /// Computes natural logarithm of (1 + x) element-wise. + /// + /// + /// + /// + public static Tensor log1p(Tensor x, string name = null) + => gen_math_ops.log1p(x, name); + + /// + /// Clips tensor values to a specified min and max. + /// + /// + /// + /// + /// + /// + public static Tensor _clip_by_value(Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = null) + => gen_math_ops._clip_by_value(t, clip_value_min, clip_value_max); + + public static Tensor sub(Tensor a, Tensor b) + => gen_math_ops.sub(a, b); + + public static Tensor sqrt(Tensor a, string name = null) + => gen_math_ops.sqrt(a, name); public static Tensor subtract(Tensor x, T[] y, string name = null) where T : struct => gen_math_ops.sub(x, ops.convert_to_tensor(y, dtype: x.dtype.as_base_dtype(), name: "y"), name); - public static Tensor multiply(Tensor x, Tensor y) => gen_math_ops.mul(x, y); + public static Tensor log(Tensor x, string name = null) + => gen_math_ops.log(x, name); + + public static Tensor equal(Tensor x, Tensor y, string name = null) + => gen_math_ops.equal(x, y, name); + + /// + /// Computes arctangent of `y/x` element-wise, respecting signs of the arguments. + /// + /// + /// + /// + /// + public static Tensor atan2(Tensor y, Tensor x, string name = null) + => gen_math_ops.atan2(y, x, name); + + /// + /// Computes the maximum of elements across dimensions of a tensor. + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor max(Tx input, Ty axis, bool keep_dims = false, string name = null) + => gen_math_ops._max(input, axis, keep_dims: keep_dims, name: name); + + /// + /// Computes the minimum of elements across dimensions of a tensor. + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor min(Tx input, Ty axis, bool keep_dims = false, string name = null) + => gen_math_ops._min(input, axis, keep_dims: keep_dims, name: name); + + /// + /// Returns the max of x and y (i.e. x > y ? x : y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor maximum(T1 x, T2 y, string name = null) + => gen_math_ops.maximum(x, y, name: name); + + /// + /// Returns the min of x and y (i.e. x < y ? x : y) element-wise. + /// + /// + /// + /// + /// + /// + /// + public static Tensor minimum(T1 x, T2 y, string name = null) + => gen_math_ops.minimum(x, y, name: name); + + public static Tensor multiply(Tensor x, Tensor y) + => gen_math_ops.mul(x, y); + + public static Tensor negative(Tensor x, string name = null) + => gen_math_ops.neg(x, name); public static Tensor divide(Tensor x, T[] y, string name = null) where T : struct => x / ops.convert_to_tensor(y, dtype: x.dtype.as_base_dtype(), name: "y"); - public static Tensor pow(T1 x, T2 y) => gen_math_ops.pow(x, y); + public static Tensor pow(T1 x, T2 y) + => gen_math_ops.pow(x, y); /// /// Computes the sum of elements across dimensions of a tensor. @@ -28,9 +242,20 @@ namespace Tensorflow /// /// /// - public static Tensor reduce_sum(Tensor input, int[] axis = null) => math_ops.reduce_sum(input); + public static Tensor reduce_sum(Tensor input, int? axis = null, int? reduction_indices = null) + { + if(!axis.HasValue && reduction_indices.HasValue) + return math_ops.reduce_sum(input, reduction_indices.Value); + return math_ops.reduce_sum(input); + } + + public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null, int? reduction_indices = null) + => math_ops.reduce_mean(input_tensor, axis: axis, keepdims: keepdims, name: name, reduction_indices: reduction_indices); public static Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) => math_ops.cast(x, dtype, name); + + public static Tensor argmax(Tensor input, int axis = -1, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64) + => gen_math_ops.arg_max(input, axis, name: name, output_type: output_type); } } diff --git a/src/TensorFlowNET.Core/APIs/tf.nn.cs b/src/TensorFlowNET.Core/APIs/tf.nn.cs index 44203906..8a1b648e 100644 --- a/src/TensorFlowNET.Core/APIs/tf.nn.cs +++ b/src/TensorFlowNET.Core/APIs/tf.nn.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Operations; using Tensorflow.Operations.Activation; namespace Tensorflow @@ -27,19 +28,39 @@ namespace Tensorflow public static IActivation relu => new relu(); - public static (Tensor, Tensor, Tensor) fused_batch_norm(Tensor x, - RefVariable scale, - RefVariable offset, - Tensor mean = null, - Tensor variance = null, - float epsilon = 0.001f, - string data_format = "NHWC", - bool is_training = true, - string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance, - epsilon: epsilon, - data_format: data_format, - is_training: is_training, - name: name); + public static Tensor[] fused_batch_norm(Tensor x, + RefVariable scale, + RefVariable offset, + Tensor mean = null, + Tensor variance = null, + float epsilon = 0.001f, + string data_format = "NHWC", + bool is_training = true, + string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance, + epsilon: epsilon, + data_format: data_format, + is_training: is_training, + name: name); + + public static IPoolFunction max_pool => new MaxPoolFunction(); + + public static Tensor[] top_k(Tensor input, int k = 1, bool sorted = true, string name = null) + => gen_nn_ops.top_kv2(input, k: k, sorted: sorted, name: name); + + public static Tensor bias_add(Tensor value, RefVariable bias, string data_format = null, string name = null) + { + return Python.with(ops.name_scope(name, "BiasAdd", new { value, bias }), scope => + { + name = scope; + return gen_nn_ops.bias_add(value, bias, data_format: data_format, name: name); + }); + } + + public static Tensor softmax(Tensor logits, int axis = -1, string name = null) + => gen_nn_ops.softmax(logits, name); + + public static Tensor softmax_cross_entropy_with_logits_v2(Tensor labels, Tensor logits, int axis = -1, string name = null) + => nn_ops.softmax_cross_entropy_with_logits_v2_helper(labels, logits, axis: axis, name: name); } } } diff --git a/src/TensorFlowNET.Core/APIs/tf.reshape.cs b/src/TensorFlowNET.Core/APIs/tf.reshape.cs index 41861968..30098581 100644 --- a/src/TensorFlowNET.Core/APIs/tf.reshape.cs +++ b/src/TensorFlowNET.Core/APIs/tf.reshape.cs @@ -10,5 +10,8 @@ namespace Tensorflow Tensor shape, string name = null) => gen_array_ops.reshape(tensor, shape, name); + public static Tensor reshape(Tensor tensor, + int[] shape, + string name = null) => gen_array_ops.reshape(tensor, shape, name); } } diff --git a/src/TensorFlowNET.Core/Clustering/KMeans.cs b/src/TensorFlowNET.Core/Clustering/KMeans.cs new file mode 100644 index 00000000..ac945cb2 --- /dev/null +++ b/src/TensorFlowNET.Core/Clustering/KMeans.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Clustering +{ + /// + /// Creates the graph for k-means clustering. + /// + public class KMeans : Python + { + public const string CLUSTERS_VAR_NAME = "clusters"; + + public const string SQUARED_EUCLIDEAN_DISTANCE = "squared_euclidean"; + public const string COSINE_DISTANCE = "cosine"; + public const string RANDOM_INIT = "random"; + public const string KMEANS_PLUS_PLUS_INIT = "kmeans_plus_plus"; + public const string KMC2_INIT = "kmc2"; + + Tensor[] _inputs; + int _num_clusters; + IInitializer _initial_clusters; + string _distance_metric; + bool _use_mini_batch; + int _mini_batch_steps_per_iteration; + int _random_seed; + int _kmeans_plus_plus_num_retries; + int _kmc2_chain_length; + + public KMeans(Tensor inputs, + int num_clusters, + IInitializer initial_clusters = null, + string distance_metric = SQUARED_EUCLIDEAN_DISTANCE, + bool use_mini_batch = false, + int mini_batch_steps_per_iteration = 1, + int random_seed = 0, + int kmeans_plus_plus_num_retries = 2, + int kmc2_chain_length = 200) + { + _inputs = new Tensor[] { inputs }; + _num_clusters = num_clusters; + _initial_clusters = initial_clusters; + _distance_metric = distance_metric; + _use_mini_batch = use_mini_batch; + _mini_batch_steps_per_iteration = mini_batch_steps_per_iteration; + _random_seed = random_seed; + _kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries; + _kmc2_chain_length = kmc2_chain_length; + } + + public object training_graph() + { + var initial_clusters = _initial_clusters; + var num_clusters = ops.convert_to_tensor(_num_clusters); + var inputs = _inputs; + _create_variables(num_clusters); + + throw new NotImplementedException("KMeans training_graph"); + } + + private RefVariable[] _create_variables(Tensor num_clusters) + { + var init_value = constant_op.constant(new float[0], dtype: TF_DataType.TF_FLOAT); + var cluster_centers = tf.Variable(init_value, name: CLUSTERS_VAR_NAME, validate_shape: false); + var cluster_centers_initialized = tf.Variable(false, dtype: TF_DataType.TF_BOOL, name: "initialized"); + RefVariable update_in_steps = null; + if (_use_mini_batch && _mini_batch_steps_per_iteration > 1) + throw new NotImplementedException("KMeans._create_variables"); + else + { + var cluster_centers_updated = cluster_centers; + var cluster_counts = _use_mini_batch ? + tf.Variable(array_ops.ones(new Tensor[] { num_clusters }, dtype: TF_DataType.TF_INT64)) : + null; + return new RefVariable[] + { + cluster_centers, + cluster_centers_initialized, + cluster_counts, + cluster_centers_updated, + update_in_steps + }; + } + } + } +} diff --git a/src/TensorFlowNET.Core/Framework/common_shapes.py.cs b/src/TensorFlowNET.Core/Framework/common_shapes.py.cs index 3fa9f6bf..70bea7b0 100644 --- a/src/TensorFlowNET.Core/Framework/common_shapes.py.cs +++ b/src/TensorFlowNET.Core/Framework/common_shapes.py.cs @@ -29,5 +29,15 @@ namespace Tensorflow.Framework { throw new NotFiniteNumberException(); } + + public static int? rank(Tensor tensor) + { + return tensor.rank; + } + + public static bool has_fully_defined_shape(Tensor tensor) + { + return tensor.getShape().is_fully_defined(); + } } } diff --git a/src/TensorFlowNET.Core/Framework/meta_graph.py.cs b/src/TensorFlowNET.Core/Framework/meta_graph.py.cs index 4e43fd7e..24be2cd6 100644 --- a/src/TensorFlowNET.Core/Framework/meta_graph.py.cs +++ b/src/TensorFlowNET.Core/Framework/meta_graph.py.cs @@ -184,7 +184,7 @@ namespace Tensorflow // Adds graph_def or the default. if (graph_def == null) - meta_graph_def.GraphDef = graph._as_graph_def(add_shapes: true); + meta_graph_def.GraphDef = graph.as_graph_def(add_shapes: true); else meta_graph_def.GraphDef = graph_def; diff --git a/src/TensorFlowNET.Core/Framework/smart_module.cs b/src/TensorFlowNET.Core/Framework/smart_module.cs index ea5bf790..57c3f67f 100644 --- a/src/TensorFlowNET.Core/Framework/smart_module.cs +++ b/src/TensorFlowNET.Core/Framework/smart_module.cs @@ -6,9 +6,9 @@ namespace Tensorflow.Framework { public class smart_module { - public static object smart_cond(Tensor pred, - Func<(Tensor, Tensor, Tensor)> true_fn = null, - Func<(Tensor, Tensor, Tensor)> false_fn = null, + public static Tensor[] smart_cond(Tensor pred, + Func true_fn = null, + Func false_fn = null, string name = null) { return control_flow_ops.cond(pred, @@ -17,9 +17,12 @@ namespace Tensorflow.Framework name: name); } - public static bool smart_constant_value(Tensor pred) + public static bool? smart_constant_value(Tensor pred) { var pred_value = tensor_util.constant_value(pred); + if (pred_value is null) + return null; + return pred_value; } } diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.py.cs b/src/TensorFlowNET.Core/Gradients/array_grad.py.cs new file mode 100644 index 00000000..cdd319ea --- /dev/null +++ b/src/TensorFlowNET.Core/Gradients/array_grad.py.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Gradients +{ + public class array_grad + { + public static Tensor[] _ReshapeGrad(Operation op, Tensor[] grads) + { + return new Tensor[] { array_ops.reshape(grads[0], array_ops.shape(op.inputs[0])), null }; + } + + public static Tensor[] _SqueezeGrad(Operation op, Tensor[] grads) + { + return new Tensor[] { _ReshapeToInput(op, grads[0]) }; + } + + private static Tensor _ReshapeToInput(Operation op, Tensor grad) + { + return array_ops.reshape(grad, array_ops.shape(op.inputs[0])); + } + + public static Tensor[] _TransposeGrad(Operation op, Tensor[] grads) + { + var p = op.inputs[1]; + return new Tensor[] { array_ops.transpose(grads[0], array_ops.invert_permutation(p)), null }; + } + } +} diff --git a/src/TensorFlowNET.Core/Gradients/control_flow_grad.py.cs b/src/TensorFlowNET.Core/Gradients/control_flow_grad.py.cs new file mode 100644 index 00000000..afc87d45 --- /dev/null +++ b/src/TensorFlowNET.Core/Gradients/control_flow_grad.py.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Operations; + +namespace Tensorflow.Gradients +{ + public class control_flow_grad + { + public static Tensor[] _MergeGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var _ = grads[1]; + var input_op = op.inputs[0].op; + var graph = ops.get_default_graph(); + var op_ctxt = control_flow_util.GetOutputContext(input_op); + var pred = (op_ctxt as CondContext).pred; + + var results = control_flow_ops._SwitchRefOrTensor(grad, pred, name: "cond_grad"); + return new Tensor[] { results.Item1, results.Item2 }; + } + } +} diff --git a/src/TensorFlowNET.Core/Gradients/gradients_impl.py.cs b/src/TensorFlowNET.Core/Gradients/gradients_impl.py.cs index abe9ef2b..028516bf 100644 --- a/src/TensorFlowNET.Core/Gradients/gradients_impl.py.cs +++ b/src/TensorFlowNET.Core/Gradients/gradients_impl.py.cs @@ -131,12 +131,23 @@ namespace Tensorflow // for ops that do not have gradients. var grad_fn = ops.get_gradient_function(op); + foreach(var (i, out_grad) in enumerate(out_grads)) + { + if(out_grad == null) + { + if (loop_state != null) + ; + else + out_grads[i] = control_flow_ops.ZerosLikeOutsideLoop(op, i); + } + } + with(ops.name_scope(op.name + "_grad"), scope1 => { string name1 = scope1; if (grad_fn != null) { - in_grads = _MaybeCompile(grad_scope, op, out_grads[0], null, grad_fn); + in_grads = _MaybeCompile(grad_scope, op, out_grads, null, grad_fn); _VerifyGeneratedGradients(in_grads, op); } @@ -226,7 +237,7 @@ namespace Tensorflow $"inputs {op.inputs._inputs.Count()}"); } - private static Tensor[] _MaybeCompile(string scope, Operation op, Tensor out_grads, Action func, Func grad_fn) + private static Tensor[] _MaybeCompile(string scope, Operation op, Tensor[] out_grads, Action func, Func grad_fn) { scope = scope.EndsWith("/") ? scope.Substring(0, scope.Length - 1) : scope; return grad_fn(op, out_grads); @@ -240,28 +251,27 @@ namespace Tensorflow private static Tensor[] _AggregatedGrads(Dictionary grads, Operation op, string gradient_uid, object loop_state, int aggregation_method = 0) { var out_grads = _GetGrads(grads, op); - for(int i = 0; i < out_grads.Length; i++) + var return_grads = new Tensor[out_grads.Length]; + + foreach(var (i, out_grad) in enumerate(out_grads)) { - var out_grad = out_grads[i]; - if(loop_state != null) + if (loop_state != null) { } - // Grads have to be Tensors or IndexedSlices - // Aggregate multiple gradients, and convert [] to None. - if(out_grad != null) + if (out_grad != null) { - if(out_grad.Length < 2) + if (out_grad.Length < 2) { string used = "nop"; - return new Tensor[] { out_grad[0] }; + return_grads[i] = out_grad[0]; } } } - return null; + return return_grads; } /// diff --git a/src/TensorFlowNET.Core/Gradients/math_grad.cs b/src/TensorFlowNET.Core/Gradients/math_grad.cs new file mode 100644 index 00000000..29faaa7a --- /dev/null +++ b/src/TensorFlowNET.Core/Gradients/math_grad.cs @@ -0,0 +1,255 @@ +//using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tensorflow.Gradients +{ + /// + /// Gradients for operators defined in math_ops.py. + /// + public class math_grad : Python + { + public static Tensor[] _AddGrad(Operation op, Tensor[] grads) + { + var x = op.inputs[0]; + var y = op.inputs[1]; + var grad = grads[0]; + if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad)) + return new Tensor[] { grad, grad }; + + var sx = array_ops.shape(x); + var sy = array_ops.shape(y); + var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); + + var sum1 = math_ops.reduce_sum(grad, rx); + var r1 = gen_array_ops.reshape(sum1, sx); + var sum2 = math_ops.reduce_sum(grad, ry); + var r2 = gen_array_ops.reshape(sum2, sy); + + return new Tensor[] { r1, r2 }; + } + + public static Tensor[] _IdGrad(Operation op, Tensor[] grads) + { + return new Tensor[] { grads[0] }; + } + + public static Tensor[] _LogGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var x = op.inputs[0]; + return with(ops.control_dependencies(new Operation[] { grad }), dp => { + x = math_ops.conj(x); + return new Tensor[] { grad * math_ops.reciprocal(x) }; + }); + } + + public static Tensor[] _MulGrad(Operation op, Tensor[] grads) + { + var x = op.inputs[0]; + var y = op.inputs[1]; + var grad = grads[0]; + if (grad is Tensor && + _ShapesFullySpecifiedAndEqual(x, y, grad) && + new TF_DataType[] { tf.int32, tf.float32 }.Contains(grad.dtype)) + return new Tensor[] { gen_math_ops.mul(grad, y), gen_math_ops.mul(grad, x) }; + + var sx = array_ops.shape(x); + var sy = array_ops.shape(y); + var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); + + x = math_ops.conj(x); + y = math_ops.conj(y); + + var mul1 = gen_math_ops.mul(grad, y); + var reduce_sum1 = math_ops.reduce_sum(mul1, rx); + var reshape1 = gen_array_ops.reshape(reduce_sum1, sx); + + var mul2 = gen_math_ops.mul(x, grad); + var reduce_sum2 = math_ops.reduce_sum(mul2, ry); + var reshape2 = gen_array_ops.reshape(reduce_sum2, sy); + + return new Tensor[] { reshape1, reshape2 }; + } + + public static Tensor[] _MatMulGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + Tensor grad_a = null, grad_b = null; + + var t_a = (bool)op.get_attr("transpose_a"); + var t_b = (bool)op.get_attr("transpose_b"); + var a = math_ops.conj(op.inputs[0]); + var b = math_ops.conj(op.inputs[1]); + if(!t_a && !t_b) + { + grad_a = gen_math_ops.mat_mul(grad, b, transpose_b: true); + grad_b = gen_math_ops.mat_mul(a, grad, transpose_a: true); + } + else if (!t_a && t_b) + { + grad_a = gen_math_ops.mat_mul(grad, b); + grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true); + } + else if (t_a && !t_b) + { + grad_a = gen_math_ops.mat_mul(grad, b); + grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true); + } + else if (t_a && t_b) + { + grad_a = gen_math_ops.mat_mul(b, grad, transpose_a: true, transpose_b: true); + grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true, transpose_b: true); + } + + return new Tensor[] { grad_a, grad_b }; + } + + public static Tensor[] _MeanGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var sum_grad = _SumGrad(op, grads)[0]; + var input_shape = op.inputs[0]._shape_tuple(); + var output_shape = op.outputs[0]._shape_tuple(); + + var input_shape_tensor = array_ops.shape(op.inputs[0]); + var output_shape_tensor = array_ops.shape(op.outputs[0]); + var factor = _safe_shape_div(math_ops.reduce_prod(input_shape_tensor), math_ops.reduce_prod(output_shape_tensor)); + + return new Tensor[] { math_ops.truediv(sum_grad, math_ops.cast(factor, sum_grad.dtype)), null }; + } + + public static Tensor[] _NegGrad(Operation op, Tensor[] grads) + { + return new Tensor[] { -grads[0] }; + } + + private static Tensor _safe_shape_div(Tensor x, Tensor y) + { + return math_ops.floordiv(x, gen_math_ops.maximum(y, 1)); + } + + public static Tensor[] _SubGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var x = op.inputs[0]; + var y = op.inputs[1]; + if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad)) + return new Tensor[] { grad, -grad }; + + var sx = array_ops.shape(x); + var sy = array_ops.shape(y); + var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); + + var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx); + var r2 = gen_array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy); + + return new Tensor[] { r1, r2 }; + } + + public static bool _ShapesFullySpecifiedAndEqual(Tensor x, Tensor y, Tensor grad) + { + var x_shape = x._shape_tuple(); + var y_shape = y._shape_tuple(); + var grad_shape = grad._shape_tuple(); + return Enumerable.SequenceEqual(x_shape, y_shape) && + Enumerable.SequenceEqual(y_shape, grad_shape) && + x.NDims != -1 && + !x_shape.Contains(-1); + } + + public static Tensor[] _SumGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var input_0_shape = op.inputs[0]._shape_tuple(); + Tensor input_shape = null; + + if (input_0_shape != null) + { + var axes = tensor_util.constant_value(op.inputs[1]); + if(!(axes is null)) + { + var rank = input_0_shape.Length; + if (Enumerable.SequenceEqual(Enumerable.Range(0, rank), axes.Data())) + { + grad = array_ops.reshape(grad, new int[] { 1 }); + if (!input_0_shape.Contains(-1)) + input_shape = constant_op.constant(input_0_shape); + else + input_shape = array_ops.shape(op.inputs[0]); + return new Tensor[] { gen_array_ops.tile(grad, input_shape), null }; + } + } + } + + input_shape = array_ops.shape(op.inputs[0]); + ops.colocate_with(input_shape); + var output_shape_kept_dims = math_ops.reduced_shape(input_shape, op.inputs[1]); + var tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims); + grad = gen_array_ops.reshape(grad, output_shape_kept_dims); + + return new Tensor[] { gen_array_ops.tile(grad, tile_scaling), null }; + } + + public static Tensor[] _RealDivGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var x = op.inputs[0]; + var y = op.inputs[1]; + + var sx = array_ops.shape(x); + var sy = array_ops.shape(y); + var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); + x = math_ops.conj(x); + y = math_ops.conj(y); + + var realdiv1 = gen_math_ops.real_div(-x, y); + var realdiv2 = gen_math_ops.real_div(realdiv1, y); + var reduce_sum1 = math_ops.reduce_sum(grad * realdiv2, ry); + var reshape1 = gen_array_ops.reshape(reduce_sum1, sy); + var realdiv3 = gen_math_ops.real_div(grad, y); + var reduce_sum2 = math_ops.reduce_sum(realdiv3, rx); + var reshape2 = gen_array_ops.reshape(reduce_sum2, sx); + + return new Tensor[] { reshape2, reshape1 }; + } + + public static Tensor[] _PowGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var x = op.inputs[0]; + var y = op.inputs[1]; + var z = op.outputs[0]; + + var sx = array_ops.shape(x); + var sy = array_ops.shape(y); + var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); + x = math_ops.conj(x); + y = math_ops.conj(y); + z = math_ops.conj(z); + var pow = gen_math_ops.pow(x, y - 1.0f); + var mul = grad * y * pow; + var reduce_sum = math_ops.reduce_sum(mul, rx); + var gx = gen_array_ops.reshape(reduce_sum, sx); + + // Avoid false singularity at x = 0 + Tensor mask = null; + if (x.dtype.is_complex()) + throw new NotImplementedException("x.dtype.is_complex()"); + else + mask = x > 0.0f; + var ones = array_ops.ones_like(x); + var safe_x = array_ops.where(mask, x, ones); + var x1 = gen_array_ops.log(safe_x); + var y1 = array_ops.zeros_like(x); + var log_x = array_ops.where(mask, x1, y1); + var mul1 = grad * z * log_x; + var reduce_sum1 = math_ops.reduce_sum(mul1, ry); + var gy = gen_array_ops.reshape(reduce_sum1, sy); + + return new Tensor[] { gx, gy }; + } + } +} diff --git a/src/TensorFlowNET.Core/Gradients/math_grad.py.cs b/src/TensorFlowNET.Core/Gradients/math_grad.py.cs deleted file mode 100644 index 00caf73d..00000000 --- a/src/TensorFlowNET.Core/Gradients/math_grad.py.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Tensorflow -{ - /// - /// Gradients for operators defined in math_ops.py. - /// - public class math_grad - { - public static (Tensor, Tensor) _AddGrad(Operation op, Tensor grad) - { - var x = op.inputs[0]; - var y = op.inputs[1]; - if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad)) - return (grad, grad); - - var sx = array_ops.shape(x); - var sy = array_ops.shape(y); - var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); - - var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx); - var r2 = gen_array_ops.reshape(math_ops.reduce_sum(grad, ry), sy); - - return (r1, r2); - } - - public static Tensor _IdGrad(Operation op, Tensor grad) - { - return grad; - } - - public static (Tensor, Tensor) _MulGrad(Operation op, Tensor grad) - { - var x = op.inputs[0]; - var y = op.inputs[1]; - if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad) && - new TF_DataType[] { tf.int32, tf.float32 }.Contains(grad.dtype)) - return (gen_math_ops.mul(grad, y), gen_math_ops.mul(grad, x)); - - var sx = array_ops.shape(x); - var sy = array_ops.shape(y); - var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); - - x = math_ops.conj(x); - y = math_ops.conj(y); - - var mul1 = gen_math_ops.mul(grad, y); - var mul2 = gen_math_ops.mul(x, grad); - var reduce_sum1 = math_ops.reduce_sum(mul1, rx); - var reduce_sum2 = math_ops.reduce_sum(mul2, ry); - var reshape1 = gen_array_ops.reshape(reduce_sum1, sx); - var reshape2 = gen_array_ops.reshape(reduce_sum2, sy); - - return (reshape1, reshape2); - } - - public static (Tensor, Tensor) _SubGrad(Operation op, Tensor grad) - { - var x = op.inputs[0]; - var y = op.inputs[1]; - if (grad is Tensor && _ShapesFullySpecifiedAndEqual(x, y, grad)) - return (grad, -grad); - - var sx = array_ops.shape(x); - var sy = array_ops.shape(y); - var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); - - var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx); - var r2 = gen_array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy); - - return (r1, r2); - } - - public static bool _ShapesFullySpecifiedAndEqual(Tensor x, Tensor y, Tensor grad) - { - return x.NDims == y.NDims && y.NDims == grad.NDims && x.NDims > -1; - } - - public static (Tensor, Tensor) _SumGrad(Operation op, Tensor grad) - { - if (op.inputs[0].NDims > -1) - { - - } - - var input_shape = array_ops.shape(op.inputs[0]); - ops.colocate_with(input_shape); - var output_shape_kept_dims = math_ops.reduced_shape(input_shape, op.inputs[1]); - var tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims); - grad = gen_array_ops.reshape(grad, output_shape_kept_dims); - - return (gen_array_ops.tile(grad, tile_scaling), null); - } - - public static Tensor _safe_shape_div(Tensor x, Tensor y) - { - return math_ops.floordiv(x, gen_math_ops.maximum(y, 1)); - } - - public static (Tensor, Tensor) _RealDivGrad(Operation op, Tensor grad) - { - var x = op.inputs[0]; - var y = op.inputs[1]; - - var sx = array_ops.shape(x); - var sy = array_ops.shape(y); - var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); - x = math_ops.conj(x); - y = math_ops.conj(y); - - var realdiv1 = gen_math_ops.real_div(-x, y); - var realdiv2 = gen_math_ops.real_div(realdiv1, y); - var reduce_sum1 = math_ops.reduce_sum(grad * realdiv2, ry); - var reshape1 = gen_array_ops.reshape(reduce_sum1, sy); - var realdiv3 = gen_math_ops.real_div(grad, y); - var reduce_sum2 = math_ops.reduce_sum(realdiv3, rx); - var reshape2 = gen_array_ops.reshape(reduce_sum2, sx); - - return (reshape2, reshape1); - } - - public static (Tensor, Tensor) _PowGrad(Operation op, Tensor grad) - { - var x = op.inputs[0]; - var y = op.inputs[1]; - var z = op.outputs[0]; - - var sx = array_ops.shape(x); - var sy = array_ops.shape(y); - var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy); - x = math_ops.conj(x); - y = math_ops.conj(y); - z = math_ops.conj(z); - var pow = gen_math_ops.pow(x, y - 1.0f); - var mul = grad * y * pow; - var reduce_sum = math_ops.reduce_sum(mul, rx); - var gx = gen_array_ops.reshape(reduce_sum, sx); - - // Avoid false singularity at x = 0 - Tensor mask = null; - if (x.dtype.is_complex()) - throw new NotImplementedException("x.dtype.is_complex()"); - else - mask = x > 0.0f; - var ones = array_ops.ones_like(x); - var safe_x = array_ops.where(mask, x, ones); - var x1 = gen_array_ops.log(safe_x); - var y1 = array_ops.zeros_like(x); - var log_x = array_ops.where(mask, x1, y1); - var mul1 = grad * z * log_x; - var reduce_sum1 = math_ops.reduce_sum(mul1, ry); - var gy = gen_array_ops.reshape(reduce_sum1, sy); - - return (gx, gy); - } - } -} diff --git a/src/TensorFlowNET.Core/Gradients/nn_grad.py.cs b/src/TensorFlowNET.Core/Gradients/nn_grad.py.cs new file mode 100644 index 00000000..0bd03046 --- /dev/null +++ b/src/TensorFlowNET.Core/Gradients/nn_grad.py.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Operations; + +namespace Tensorflow.Gradients +{ + public class nn_grad + { + /// + /// Return the gradients for the 2 inputs of bias_op. + /// + /// + /// + /// + public static Tensor[] _BiasAddGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + string data_format = op.get_attr("data_format")?.ToString(); + var bias_add_grad = gen_nn_ops.bias_add_grad(out_backprop: grad, data_format: data_format); + return new Tensor[] { grad, bias_add_grad }; + } + + public static Tensor[] _ReluGrad(Operation op, Tensor[] grads) + { + return new Tensor[] { gen_nn_ops.relu_grad(grads[0], op.outputs[0]) }; + } + + /// + /// The derivative of the softmax nonlinearity. + /// + /// + /// + /// + public static Tensor[] _SoftmaxGrad(Operation op, Tensor[] grads) + { + var grad_softmax = grads[0]; + + var softmax = op.outputs[0]; + var mul = grad_softmax * softmax; + var sum_channels = math_ops.reduce_sum(mul, -1, keepdims: true); + var sub = grad_softmax - sum_channels; + return new Tensor[] { sub * softmax }; + } + + /// + /// Gradient function for SoftmaxCrossEntropyWithLogits. + /// + /// + /// + /// + /// + public static Tensor[] _SoftmaxCrossEntropyWithLogitsGrad(Operation op, Tensor[] grads) + { + var grad_loss = grads[0]; + var grad_grad = grads[1]; + var softmax_grad = op.outputs[1]; + var grad = _BroadcastMul(grad_loss, softmax_grad); + + var logits = op.inputs[0]; + if(grad_grad != null && !IsZero(grad_grad)) + { + throw new NotImplementedException("_SoftmaxCrossEntropyWithLogitsGrad"); + } + + return new Tensor[] + { + grad, + _BroadcastMul(grad_loss, -nn_ops.log_softmax(logits)) + }; + } + + private static bool IsZero(Tensor g) + { + if (new string[] { "ZerosLike", "Zeros" }.Contains(g.op.type)) + return true; + + throw new NotImplementedException("IsZero"); + } + + private static Tensor _BroadcastMul(Tensor vec, Tensor mat) + { + vec = array_ops.expand_dims(vec, -1); + return vec * mat; + } + + /// + /// Return the gradients for TopK. + /// + /// + /// + /// + public static Tensor[] _TopKGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var _ = grads[1]; + + var in_shape = array_ops.shape(op.inputs[0]); + var ind_shape = array_ops.shape(op.outputs[1]); + + // int32 is not supported on GPU hence up-casting + var cast = math_ops.cast(ind_shape, TF_DataType.TF_INT64); + var size = array_ops.size(ind_shape) - 1; + var ind_lastdim = array_ops.gather(cast, size); + + // Flatten indices to 2D. + var stack = array_ops.stack(new object[] { -1L, ind_lastdim }); + var ind_2d = array_ops.reshape(op.outputs[1], stack); + + var in_lastdim = array_ops.gather(math_ops.cast(in_shape, TF_DataType.TF_INT64), + array_ops.size(in_shape) - 1); + var outerdim = array_ops.shape(ind_2d)[0]; + + // Compute linear indices(flattened to 1D). + var cast1 = math_ops.cast(outerdim, TF_DataType.TF_INT64); + var range2 = math_ops.range(0L, cast1 * in_lastdim, in_lastdim); + var dim2 = array_ops.expand_dims(range2, -1); + var cast2 = math_ops.cast(dim2, TF_DataType.TF_INT32); + var ind = array_ops.reshape(ind_2d + cast2, new int[] { -1 }); + + // Substitute grad to appropriate locations and fill the rest with zeros, + // finally reshaping it to the original input shape. + var scatter = gen_array_ops.scatter_nd(array_ops.expand_dims(ind, -1), + array_ops.reshape(grad, new int[] { -1 }), + new Tensor[] { math_ops.reduce_prod(in_shape) }); + + return new Tensor[] + { + array_ops.reshape(scatter, in_shape), + array_ops.zeros(new int[0], dtype: TF_DataType.TF_INT32) + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs b/src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs new file mode 100644 index 00000000..7b650d00 --- /dev/null +++ b/src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Gradients; + +namespace Tensorflow +{ + public partial class ops + { + public static Func get_gradient_function(Operation op) + { + if (op.inputs == null) return null; + + // map tensorflow\python\ops\math_grad.py + return (oper, out_grads) => + { + // Console.WriteLine($"get_gradient_function: {oper.type} '{oper.name}'"); + + switch (oper.type) + { + case "Add": + return math_grad._AddGrad(oper, out_grads); + case "BiasAdd": + return nn_grad._BiasAddGrad(oper, out_grads); + case "Identity": + return math_grad._IdGrad(oper, out_grads); + case "Log": + return math_grad._LogGrad(oper, out_grads); + case "MatMul": + return math_grad._MatMulGrad(oper, out_grads); + case "Merge": + return control_flow_grad._MergeGrad(oper, out_grads); + case "Mul": + return math_grad._MulGrad(oper, out_grads); + case "Mean": + return math_grad._MeanGrad(oper, out_grads); + case "Neg": + return math_grad._NegGrad(oper, out_grads); + case "Sum": + return math_grad._SumGrad(oper, out_grads); + case "Sub": + return math_grad._SubGrad(oper, out_grads); + case "Pow": + return math_grad._PowGrad(oper, out_grads); + case "RealDiv": + return math_grad._RealDivGrad(oper, out_grads); + case "Reshape": + return array_grad._ReshapeGrad(oper, out_grads); + case "Relu": + return nn_grad._ReluGrad(oper, out_grads); + case "Squeeze": + return array_grad._SqueezeGrad(oper, out_grads); + case "Softmax": + return nn_grad._SoftmaxGrad(oper, out_grads); + case "SoftmaxCrossEntropyWithLogits": + return nn_grad._SoftmaxCrossEntropyWithLogitsGrad(oper, out_grads); + case "Transpose": + return array_grad._TransposeGrad(oper, out_grads); + case "TopK": + case "TopKV2": + return nn_grad._TopKGrad(oper, out_grads); + default: + throw new NotImplementedException($"get_gradient_function {oper.type}"); + } + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Graphs/FreezeGraph.cs b/src/TensorFlowNET.Core/Graphs/FreezeGraph.cs new file mode 100644 index 00000000..a77b4f3a --- /dev/null +++ b/src/TensorFlowNET.Core/Graphs/FreezeGraph.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow +{ + public class FreezeGraph + { + public static void freeze_graph(string input_graph, + string input_saver, + bool input_binary, + string input_checkpoint, + string output_node_names, + string restore_op_name, + string filename_tensor_name, + string output_graph, + bool clear_devices, + string initializer_nodes) + { + + } + } +} diff --git a/src/TensorFlowNET.Core/Graphs/Graph.Export.cs b/src/TensorFlowNET.Core/Graphs/Graph.Export.cs index 6501de70..0ca80be3 100644 --- a/src/TensorFlowNET.Core/Graphs/Graph.Export.cs +++ b/src/TensorFlowNET.Core/Graphs/Graph.Export.cs @@ -18,7 +18,7 @@ namespace Tensorflow return buffer; } - public GraphDef _as_graph_def(bool add_shapes = false) + private GraphDef _as_graph_def(bool add_shapes = false) { var buffer = ToGraphDef(Status); Status.Check(); @@ -30,5 +30,8 @@ namespace Tensorflow return def; } + + public GraphDef as_graph_def(bool add_shapes = false) + => _as_graph_def(add_shapes); } } diff --git a/src/TensorFlowNET.Core/Graphs/Graph.cs b/src/TensorFlowNET.Core/Graphs/Graph.cs index 71faa045..916c42a7 100644 --- a/src/TensorFlowNET.Core/Graphs/Graph.cs +++ b/src/TensorFlowNET.Core/Graphs/Graph.cs @@ -355,7 +355,7 @@ namespace Tensorflow return _collections.Keys.Where(x => !x.StartsWith("__")).ToArray(); } - public object get_collection(string name, string scope = "") + public object get_collection(string name, string scope = null) { return _collections.ContainsKey(name) ? _collections[name] : null; } diff --git a/src/TensorFlowNET.Core/Graphs/graph_io.py.cs b/src/TensorFlowNET.Core/Graphs/graph_io.py.cs index 31f33221..7abc4cab 100644 --- a/src/TensorFlowNET.Core/Graphs/graph_io.py.cs +++ b/src/TensorFlowNET.Core/Graphs/graph_io.py.cs @@ -10,7 +10,7 @@ namespace Tensorflow { public static string write_graph(Graph graph, string logdir, string name, bool as_text = true) { - var graph_def = graph._as_graph_def(); + var graph_def = graph.as_graph_def(); string path = Path.Combine(logdir, name); if (as_text) File.WriteAllText(path, graph_def.ToString()); diff --git a/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs b/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs index bfd96d6a..c4980e92 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs @@ -9,17 +9,20 @@ namespace Tensorflow.Keras.Engine /// public class InputSpec { - public int ndim; + public int? ndim; + public int? min_ndim; Dictionary axes; - public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, + public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, int? ndim = null, + int? min_ndim = null, Dictionary axes = null) { - this.ndim = ndim.Value; + this.ndim = ndim; if (axes == null) axes = new Dictionary(); this.axes = axes; + this.min_ndim = min_ndim; } } } diff --git a/src/TensorFlowNET.Core/Keras/Engine/Model.cs b/src/TensorFlowNET.Core/Keras/Engine/Model.cs index a0ad4a53..697a1938 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/Model.cs @@ -4,7 +4,12 @@ using System.Text; namespace Tensorflow.Keras.Engine { - internal class Model : Network + public class Model : Network { + public Model(string name = null) + : base(name: name) + { + + } } } diff --git a/src/TensorFlowNET.Core/Keras/Engine/Network.cs b/src/TensorFlowNET.Core/Keras/Engine/Network.cs index 6eff46c4..e06d4e05 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/Network.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/Network.cs @@ -1,10 +1,41 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Keras.Layers; namespace Tensorflow.Keras.Engine { public class Network : Layer { + protected bool _is_compiled; + protected bool _expects_training_arg; + protected bool _compute_output_and_mask_jointly; + /// + /// All layers in order of horizontal graph traversal. + /// Entries are unique. Includes input and output layers. + /// + protected List _layers; + + public Network(string name = null) + : base(name: name) + { + _init_subclassed_network(name); + } + + protected virtual void _init_subclassed_network(string name = null) + { + _base_init(name: name); + } + + protected virtual void _base_init(string name = null) + { + _init_set_name(name); + trainable = true; + _is_compiled = false; + _expects_training_arg = false; + _compute_output_and_mask_jointly = false; + supports_masking = false; + _layers = new List(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs b/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs index d3762bfb..587a956a 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs @@ -1,24 +1,56 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Keras.Layers; namespace Tensorflow.Keras.Engine { - public class Sequential : Network, IPython + public class Sequential : Model, IPython { - public void Dispose() + public Sequential(string name = null) + : base(name: name) { - throw new NotImplementedException(); + supports_masking = true; + _compute_output_and_mask_jointly = true; } public void __enter__() { - throw new NotImplementedException(); + + } + + public void add(Layer layer) + { + built = false; + var set_inputs = false; + if(_layers.Count == 0) + { + var (batch_shape, dtype) = (layer._batch_input_shape, layer._dtype); + if(batch_shape != null) + { + // Instantiate an input layer. + var x = keras.layers.Input( + batch_shape: batch_shape, + dtype: dtype, + name: layer._name + "_input"); + + // This will build the current layer + // and create the node connecting the current layer + // to the input layer we just created. + layer.__call__(x); + set_inputs = true; + } + } } public void __exit__() { - throw new NotImplementedException(); + + } + + public void Dispose() + { + } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs b/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs index 1223e350..c93c07c0 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs @@ -3,11 +3,10 @@ using System.Collections.Generic; using System.Linq; using System.Text; using Tensorflow.Keras.Utils; -using Tensorflow.Layers; namespace Tensorflow.Keras.Layers { - public class BatchNormalization : Layer + public class BatchNormalization : Tensorflow.Layers.Layer { private bool _USE_V2_BEHAVIOR = true; private float momentum; @@ -132,6 +131,7 @@ namespace Tensorflow.Keras.Layers if (fused) { outputs = _fused_batch_norm(inputs, training: training); + return outputs; } throw new NotImplementedException("BatchNormalization call"); @@ -142,7 +142,7 @@ namespace Tensorflow.Keras.Layers var beta = this.beta; var gamma = this.gamma; - Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_training = () => + Func _fused_batch_norm_training = () => { return tf.nn.fused_batch_norm( inputs, @@ -152,7 +152,7 @@ namespace Tensorflow.Keras.Layers data_format: _data_format); }; - Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_inference = () => + Func _fused_batch_norm_inference = () => { return tf.nn.fused_batch_norm( inputs, @@ -165,9 +165,41 @@ namespace Tensorflow.Keras.Layers data_format: _data_format); }; - tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference); + var results = tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference); + var (output, mean, variance) = (results[0], results[1], results[2]); + var training_value = tf_utils.constant_value(training); - throw new NotImplementedException("_fused_batch_norm"); + Tensor momentum_tensor; + if (training_value == null) + { + momentum_tensor = tf_utils.smart_cond(training, + () => new float[] { momentum }, () => new float[] { 1.0f })[0]; + } + else + { + momentum_tensor = ops.convert_to_tensor(momentum); + } + + if(training_value == null) + { + var mean_update = _assign_moving_average(moving_mean, mean, momentum_tensor); + var variance_update = _assign_moving_average(moving_variance, variance, momentum_tensor); + add_update(new Tensor[] { mean_update }, inputs: true); + add_update(new Tensor[] { variance_update }, inputs: true); + } + + return output; + } + + public Tensor _assign_moving_average(RefVariable variable, Tensor value, Tensor momentum) + { + return Python.with(ops.name_scope(null, "AssignMovingAvg", new { variable, value, momentum }), scope => + { + // var cm = ops.colocate_with(variable); + var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay"); + var update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay; + return state_ops.assign_sub(variable, update_delta, name: scope); + }); } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/Dense.cs b/src/TensorFlowNET.Core/Keras/Layers/Dense.cs new file mode 100644 index 00000000..9a3b45ba --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Dense.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Keras.Engine; +using Tensorflow.Operations.Activation; +using static Tensorflow.tf; + +namespace Tensorflow.Keras.Layers +{ + public class Dense : Tensorflow.Layers.Layer + { + protected int units; + protected IActivation activation; + protected bool use_bias; + protected IInitializer kernel_initializer; + protected IInitializer bias_initializer; + protected RefVariable kernel; + protected RefVariable bias; + + public Dense(int units, + IActivation activation, + bool use_bias = true, + bool trainable = false, + IInitializer kernel_initializer = null, + IInitializer bias_initializer = null) : base(trainable: trainable) + { + this.units = units; + this.activation = activation; + this.use_bias = use_bias; + this.kernel_initializer = kernel_initializer; + this.bias_initializer = bias_initializer; + this.supports_masking = true; + this.input_spec = new InputSpec(min_ndim: 2); + } + + protected override void build(TensorShape input_shape) + { + var last_dim = input_shape.Dimensions.Last(); + var axes = new Dictionary(); + axes[-1] = last_dim; + input_spec = new InputSpec(min_ndim: 2, axes: axes); + kernel = add_weight( + "kernel", + shape: new int[] { last_dim, units }, + initializer: kernel_initializer, + dtype: _dtype, + trainable: true); + if (use_bias) + bias = add_weight( + "bias", + shape: new int[] { units }, + initializer: bias_initializer, + dtype: _dtype, + trainable: true); + + built = true; + } + + protected override Tensor call(Tensor inputs, Tensor training = null) + { + Tensor outputs = null; + var rank = inputs.rank; + if(rank > 2) + { + throw new NotImplementedException("call rank > 2"); + } + else + { + outputs = gen_math_ops.mat_mul(inputs, kernel); + } + + if (use_bias) + outputs = nn.bias_add(outputs, bias); + if (activation != null) + return activation.Activate(outputs); + + return outputs; + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs b/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs new file mode 100644 index 00000000..3ea6d65d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Layers +{ + public class Embedding : Layer + { + private int input_dim; + private int output_dim; + private bool mask_zero; + public RefVariable embeddings; + public IInitializer embeddings_initializer; + + public Embedding(int input_dim, int output_dim, + IInitializer embeddings_initializer = null, + bool mask_zero = false, + TF_DataType dtype = TF_DataType.TF_FLOAT, + int[] input_shape = null) : base(dtype: dtype, input_shape: input_shape) + { + this.input_dim = input_dim; + this.output_dim = output_dim; + this.embeddings_initializer = embeddings_initializer == null ? tf.uniform_initializer : embeddings_initializer; + this.mask_zero = mask_zero; + supports_masking = mask_zero; + } + + protected override void build(TensorShape input_shape) + { + embeddings = add_weight(shape: new int[] { input_dim, output_dim }, + initializer: embeddings_initializer, + name: "embeddings"); + built = true; + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/IPoolFunction.cs b/src/TensorFlowNET.Core/Keras/Layers/IPoolFunction.cs new file mode 100644 index 00000000..07544f10 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/IPoolFunction.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow +{ + public interface IPoolFunction + { + Tensor Apply(Tensor value, + int[] ksize, + int[] strides, + string padding, + string data_format = "NHWC", + string name = null); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs b/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs new file mode 100644 index 00000000..60257ee0 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Layer to be used as an entry point into a Network (a graph of layers). + /// + public class InputLayer : Layer + { + public bool sparse; + public int? batch_size; + public bool is_placeholder; + + public InputLayer(int[] input_shape = null, + int? batch_size = null, + TF_DataType dtype = TF_DataType.DtInvalid, + string name = null, + bool sparse = false, + Tensor input_tensor = null) + { + built = true; + this.sparse = sparse; + this.batch_size = batch_size; + this.supports_masking = true; + + if (input_tensor == null) + { + var batch_input_shape = new int[] { batch_size.HasValue ? batch_size.Value : -1, -1 }; + + if (sparse) + { + throw new NotImplementedException("InputLayer sparse is true"); + } + else + { + input_tensor = backend.placeholder( + shape: batch_input_shape, + dtype: dtype, + name: name); + } + + is_placeholder = true; + _batch_input_shape = batch_input_shape; + } + + new Node(this, + inbound_layers: new Layer[0], + node_indices: new int[0], + tensor_indices: new int[0], + input_tensors: new Tensor[] { input_tensor }, + output_tensors: new Tensor[] { input_tensor }); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Engine/Layer.cs b/src/TensorFlowNET.Core/Keras/Layers/Layer.cs similarity index 72% rename from src/TensorFlowNET.Core/Keras/Engine/Layer.cs rename to src/TensorFlowNET.Core/Keras/Layers/Layer.cs index 2e442e65..4b8eebba 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Layer.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; +using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Engine +namespace Tensorflow.Keras.Layers { /// /// Base layer class. @@ -19,7 +21,7 @@ namespace Tensorflow.Keras.Engine /// protected bool built; protected bool trainable; - protected TF_DataType _dtype; + public TF_DataType _dtype; /// /// A stateful layer is a layer whose updates are run during inference too, /// for instance stateful RNNs. @@ -31,11 +33,22 @@ namespace Tensorflow.Keras.Engine protected InputSpec input_spec; protected bool supports_masking; protected List _trainable_weights; - protected string _name; + public string _name; protected string _base_name; protected bool _compute_previous_mask; + protected List _updates; + public int[] _batch_input_shape; - public Layer(bool trainable = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid) + private List _inbound_nodes; + public List inbound_nodes => _inbound_nodes; + + private List _outbound_nodes; + public List outbound_nodes => _outbound_nodes; + + public Layer(bool trainable = true, + string name = null, + TF_DataType dtype = TF_DataType.DtInvalid, + int[] input_shape = null) { this.trainable = trainable; this._dtype = dtype; @@ -45,13 +58,22 @@ namespace Tensorflow.Keras.Engine _init_set_name(name); _trainable_weights = new List(); _compute_previous_mask = false; + _updates = new List(); + + // Manage input shape information if passed. + + _batch_input_shape = new int[] { -1, -1 }; + + _dtype = dtype; + + _inbound_nodes = new List(); } - public Tensor __call__(Tensor inputs, + public Tensor __call__(Tensor[] inputs, Tensor training = null, VariableScope scope = null) { - var input_list = new Tensor[] { inputs }; + var input_list = inputs; Tensor outputs = null; // We will attempt to build a TF graph if & only if all inputs are symbolic. @@ -74,9 +96,9 @@ namespace Tensorflow.Keras.Engine // Symbolic execution on symbolic tensors. We will attempt to build // the corresponding TF subgraph inside `backend.get_graph()` var graph = backend.get_graph(); - outputs = call(inputs, training: training); - _handle_activity_regularization(inputs, outputs); - _set_mask_metadata(inputs, outputs, null); + outputs = call(inputs[0], training: training); + _handle_activity_regularization(inputs[0], outputs); + _set_mask_metadata(inputs[0], outputs, null); } }); @@ -103,7 +125,7 @@ namespace Tensorflow.Keras.Engine protected virtual Tensor call(Tensor inputs, Tensor training = null) { - throw new NotImplementedException("Layer.call"); + return inputs; } protected virtual string _name_scope() @@ -111,15 +133,15 @@ namespace Tensorflow.Keras.Engine return null; } - protected void _maybe_build(Tensor inputs) + protected void _maybe_build(Tensor[] inputs) { - var input_list = new Tensor[] { inputs }; - build(inputs.getShape()); + var input_list = inputs; + build(input_list[0].getShape()); } protected virtual void build(TensorShape input_shape) { - throw new NotImplementedException("Layer.build"); + built = true; } protected virtual RefVariable add_weight(string name, @@ -129,10 +151,16 @@ namespace Tensorflow.Keras.Engine bool? trainable = null, Func getter = null) { + if (dtype == TF_DataType.DtInvalid) + dtype = TF_DataType.TF_FLOAT; + + if (trainable == null) + trainable = true; + var variable = _add_variable_with_custom_getter(name, shape, dtype: dtype, - getter: getter, + getter: getter == null ? base_layer_utils.make_variable : getter, overwrite: true, initializer: initializer, trainable: trainable.Value); @@ -142,6 +170,12 @@ namespace Tensorflow.Keras.Engine return variable; } + protected virtual void add_update(Tensor[] updates, bool inputs = false) + { + var updates_op = updates.Select(x => x.op).ToArray(); + _updates.AddRange(updates_op); + } + protected virtual void _init_set_name(string name) { string base_name = name; diff --git a/src/TensorFlowNET.Core/Keras/Layers/MaxPooling2D.cs b/src/TensorFlowNET.Core/Keras/Layers/MaxPooling2D.cs new file mode 100644 index 00000000..649c1a33 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/MaxPooling2D.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Text; +using static Tensorflow.tf; + +namespace Tensorflow.Keras.Layers +{ + public class MaxPooling2D : Pooling2D + { + public MaxPooling2D( + int[] pool_size, + int[] strides, + string padding = "valid", + string data_format = null, + string name = null) : base(nn.max_pool, pool_size, + strides, + padding: padding, + data_format: data_format, + name: name) + { + + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/Node.cs b/src/TensorFlowNET.Core/Keras/Layers/Node.cs new file mode 100644 index 00000000..d4144c62 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Node.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tensorflow.Keras.Layers +{ + /// + /// A `Node` describes the connectivity between two layers. + /// + public class Node + { + public InputLayer outbound_layer; + public Layer[] inbound_layers; + public int[] node_indices; + public int[] tensor_indices; + public Tensor[] input_tensors; + public Tensor[] output_tensors; + public int[][] input_shapes; + public int[][] output_shapes; + + /// + /// + /// + /// + /// the layer that takes + /// `input_tensors` and turns them into `output_tensors` + /// (the node gets created when the `call` + /// method of the layer was called). + /// + /// + /// a list of layers, the same length as `input_tensors`, + /// the layers from where `input_tensors` originate. + /// + /// + /// a list of integers, the same length as `inbound_layers`. + /// `node_indices[i]` is the origin node of `input_tensors[i]` + /// (necessary since each inbound layer might have several nodes, + /// e.g. if the layer is being shared with a different data stream). + /// + /// + /// list of input tensors. + /// list of output tensors. + public Node(InputLayer outbound_layer, + Layer[] inbound_layers, + int[] node_indices, + int[] tensor_indices, + Tensor[] input_tensors, + Tensor[] output_tensors) + { + this.outbound_layer = outbound_layer; + this.inbound_layers = inbound_layers; + this.node_indices = node_indices; + this.tensor_indices = tensor_indices; + this.input_tensors = input_tensors; + this.output_tensors = output_tensors; + + input_shapes = input_tensors.Select(x => x._shape_tuple()).ToArray(); + output_shapes = output_tensors.Select(x => x._shape_tuple()).ToArray(); + + // Add nodes to all layers involved. + foreach (var layer in inbound_layers) + { + if (layer != null) + layer.outbound_nodes.Add(this); + } + + outbound_layer.inbound_nodes.Add(this); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs b/src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs new file mode 100644 index 00000000..69c4d65c --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Utils; + +namespace Tensorflow.Keras.Layers +{ + public class Pooling2D : Tensorflow.Layers.Layer + { + private IPoolFunction pool_function; + private int[] pool_size; + private int[] strides; + private string padding; + private string data_format; + private InputSpec input_spec; + + public Pooling2D(IPoolFunction pool_function, + int[] pool_size, + int[] strides, + string padding = "valid", + string data_format = null, + string name = null) : base(name: name) + { + this.pool_function = pool_function; + this.pool_size = conv_utils.normalize_tuple(pool_size, 2, "pool_size"); + this.strides = conv_utils.normalize_tuple(strides, 2, "strides"); + this.padding = conv_utils.normalize_padding(padding); + this.data_format = conv_utils.normalize_data_format(data_format); + this.input_spec = new InputSpec(ndim: 4); + } + + protected override Tensor call(Tensor inputs, Tensor training = null) + { + int[] pool_shape; + if (data_format == "channels_last") + { + pool_shape = new int[] { 1, pool_size[0], pool_size[1], 1 }; + strides = new int[] { 1, strides[0], strides[1], 1 }; + } + else + { + pool_shape = new int[] { 1, 1, pool_size[0], pool_size[1] }; + strides = new int[] { 1, 1, strides[0], strides[1] }; + } + + var outputs = pool_function.Apply( + inputs, + ksize: pool_shape, + strides: strides, + padding: padding.ToUpper(), + data_format: conv_utils.convert_data_format(data_format, 4)); + + return outputs; + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs b/src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs similarity index 66% rename from src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs rename to src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs index 1f397425..682760f0 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs +++ b/src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs @@ -2,10 +2,19 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.Engine +namespace Tensorflow.Keras.Utils { public class base_layer_utils { + public static RefVariable make_variable(string name, + int[] shape, + TF_DataType dtype = TF_DataType.TF_FLOAT, + IInitializer initializer = null, + bool trainable = false) + { + throw new NotImplementedException(""); + } + /// /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph. /// diff --git a/src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs b/src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs index ef348d1b..790470ee 100644 --- a/src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs +++ b/src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs @@ -29,5 +29,20 @@ namespace Tensorflow.Keras.Utils else throw new ValueError($"Invalid data_format: {data_format}"); } + + public static int[] normalize_tuple(int[] value, int n, string name) + { + return value; + } + + public static string normalize_padding(string value) + { + return value.ToLower(); + } + + public static string normalize_data_format(string value) + { + return value.ToLower(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs b/src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs index 4e155493..c57344c2 100644 --- a/src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs +++ b/src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs @@ -13,14 +13,19 @@ namespace Tensorflow.Keras.Utils return tensors.Select(x => is_symbolic_tensor(x)).Count() == tensors.Length; } + public static bool? constant_value(Tensor pred) + { + return smart_module.smart_constant_value(pred); + } + public static bool is_symbolic_tensor(Tensor tensor) { return true; } - public static object smart_cond(Tensor pred, - Func<(Tensor, Tensor, Tensor)> true_fn = null, - Func<(Tensor, Tensor, Tensor)> false_fn = null, + public static Tensor[] smart_cond(Tensor pred, + Func true_fn = null, + Func false_fn = null, string name = null) { return smart_module.smart_cond(pred, diff --git a/src/TensorFlowNET.Core/Keras/backend.cs b/src/TensorFlowNET.Core/Keras/backend.cs index 0196bfea..17ab0fbb 100644 --- a/src/TensorFlowNET.Core/Keras/backend.cs +++ b/src/TensorFlowNET.Core/Keras/backend.cs @@ -11,6 +11,22 @@ namespace Tensorflow.Keras } + public static Tensor placeholder(int[] shape = null, + int ndim = -1, + TF_DataType dtype = TF_DataType.DtInvalid, + bool sparse = false, + string name = null) + { + if(sparse) + { + throw new NotImplementedException("placeholder sparse is true"); + } + else + { + return gen_array_ops.placeholder(dtype: dtype, shape: new TensorShape(shape), name: name); + } + } + public static Graph get_graph() { return ops.get_default_graph(); diff --git a/src/TensorFlowNET.Core/Layers/Dense.cs b/src/TensorFlowNET.Core/Layers/Dense.cs new file mode 100644 index 00000000..e2868877 --- /dev/null +++ b/src/TensorFlowNET.Core/Layers/Dense.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Operations.Activation; + +namespace Tensorflow.Layers +{ + public class Dense : Keras.Layers.Dense + { + public Dense(int units, + IActivation activation, + bool use_bias = true, + bool trainable = false, + IInitializer kernel_initializer = null) : base(units, + activation, + use_bias: use_bias, + trainable: trainable, + kernel_initializer: kernel_initializer) + { + + } + } +} diff --git a/src/TensorFlowNET.Core/Layers/Layer.cs b/src/TensorFlowNET.Core/Layers/Layer.cs index 1ca856f0..4b2d9cf3 100644 --- a/src/TensorFlowNET.Core/Layers/Layer.cs +++ b/src/TensorFlowNET.Core/Layers/Layer.cs @@ -1,11 +1,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; -using Tensorflow.Keras.Engine; namespace Tensorflow.Layers { - public class Layer : Keras.Engine.Layer + public class Layer : Keras.Layers.Layer { protected Graph _graph; @@ -52,14 +52,26 @@ namespace Tensorflow.Layers Python.with(scope_context_manager, scope2 => _current_scope = scope2); // Actually call layer - var outputs = base.__call__(inputs, training: training); + var outputs = base.__call__(new Tensor[] { inputs }, training: training); // Update global default collections. - //_add_elements_to_collection(updates, ops.GraphKeys.UPDATE_OPS); + _add_elements_to_collection(_updates.ToArray(), new string[] { ops.GraphKeys.UPDATE_OPS }); return outputs; } + protected virtual void _add_elements_to_collection(Operation[] elements, string[] collection_list) + { + foreach(var name in collection_list) + { + var collection = ops.get_collection_ref(name) as List; + + foreach (var element in elements) + if (!collection.Contains(element)) + collection.Add(element); + } + } + protected virtual RefVariable add_weight(string name, int[] shape, TF_DataType dtype = TF_DataType.DtInvalid, diff --git a/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs b/src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs similarity index 100% rename from src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs rename to src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/CondContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/CondContext.cs index 3c233e8d..c1a87224 100644 --- a/src/TensorFlowNET.Core/Operations/ControlFlows/CondContext.cs +++ b/src/TensorFlowNET.Core/Operations/ControlFlows/CondContext.cs @@ -10,22 +10,23 @@ namespace Tensorflow.Operations public class CondContext : ControlFlowContext { private string _name; + /// /// The boolean tensor for the cond predicate /// private Tensor _pred; - /// - /// The predicate tensor in this branch - /// - private Tensor _pivot; + public Tensor pred => _pred; + /// /// 0 or 1 representing this branch /// private int _branch; + /// /// /// private List _values = new List(); + private Dictionary _external_values = new Dictionary(); /// @@ -63,14 +64,23 @@ namespace Tensorflow.Operations } } - public (Tensor, Tensor, Tensor) BuildCondBranch(Func<(Tensor, Tensor, Tensor)> fn) + public (T[], Tensor[]) BuildCondBranch(Func fn) { // Add the subgraph defined by fn() to the graph. var pre_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION); var original_result = fn(); var post_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION); - return original_result; + switch (original_result) + { + case Tensor[] results: + return (original_result, results); + case float[] fv: + var result = ops.convert_to_tensor(fv[0]); + return (original_result, new Tensor[] { result }); + default: + return (original_result, new Tensor[0]); + } } } } diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs index 7079606f..8776f171 100644 --- a/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs +++ b/src/TensorFlowNET.Core/Operations/ControlFlows/ControlFlowContext.cs @@ -6,6 +6,11 @@ namespace Tensorflow.Operations { public abstract class ControlFlowContext : IPython, IControlFlowContext { + /// + /// The predicate tensor in this branch + /// + protected Tensor _pivot; + protected Stack _context_stack; public ControlFlowContext() { @@ -28,6 +33,29 @@ namespace Tensorflow.Operations graph._set_control_flow_context(this); } + public void AddOp(Operation op) + { + _AddOpInternal(op); + } + + protected virtual void _AddOpInternal(Operation op) + { + if(op.inputs.Length == 0) + { + _RemoveExternalControlEdges(op); + op._add_control_input(_pivot.op); + } + else + { + + } + } + + protected virtual void _RemoveExternalControlEdges(Operation op) + { + var internal_control_inputs = op.control_inputs; + } + public void Exit() { var graph = ops.get_default_graph(); diff --git a/src/TensorFlowNET.Core/Operations/ControlFlows/IControlFlowContext.cs b/src/TensorFlowNET.Core/Operations/ControlFlows/IControlFlowContext.cs index 52719538..6bd8c6e2 100644 --- a/src/TensorFlowNET.Core/Operations/ControlFlows/IControlFlowContext.cs +++ b/src/TensorFlowNET.Core/Operations/ControlFlows/IControlFlowContext.cs @@ -6,5 +6,6 @@ namespace Tensorflow { public interface IControlFlowContext { + void AddOp(Operation op); } } diff --git a/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs b/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs new file mode 100644 index 00000000..2055bf83 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Operations.Initializers +{ + public class RandomUniform : IInitializer + { + private int? seed; + private float minval; + private float maxval; + private TF_DataType dtype; + + public RandomUniform() + { + + } + + public Tensor call(TensorShape shape, TF_DataType dtype = TF_DataType.DtInvalid) + { + return random_ops.random_uniform(shape, + minval: minval, + maxval: maxval, + dtype: dtype, + seed: seed); + } + + public object get_config() + { + return new { + minval, + maxval, + seed, + dtype + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs b/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs index 4c0a7cee..ae639c8e 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs @@ -11,9 +11,9 @@ namespace Tensorflow.Operations.Initializers private int? seed; private TF_DataType dtype; - public TruncatedNormal(float mean = 0.0f, - float stddev = 1.0f, - int? seed = null, + public TruncatedNormal(float mean = 0.0f, + float stddev = 1.0f, + int? seed = null, TF_DataType dtype = TF_DataType.TF_FLOAT) { this.mean = mean; @@ -24,7 +24,7 @@ namespace Tensorflow.Operations.Initializers public Tensor call(TensorShape shape, TF_DataType dtype) { - throw new NotImplementedException(""); + return random_ops.truncated_normal(shape, mean, stddev, dtype : dtype, seed: seed); } public object get_config() diff --git a/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs b/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs new file mode 100644 index 00000000..5f15706e --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Operations +{ + public class MaxPoolFunction : Python, IPoolFunction + { + public Tensor Apply(Tensor value, + int[] ksize, + int[] strides, + string padding, + string data_format = "NHWC", + string name = null) + { + return with(ops.name_scope(name, "MaxPool", new { value }), scope => { + + value = ops.convert_to_tensor(value, name: "input"); + return gen_nn_ops.max_pool( + value, + ksize: ksize, + strides: strides, + padding: padding, + data_format: data_format, + name: name); + }); + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs index a93c1653..72ca1c1b 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs @@ -53,7 +53,23 @@ namespace Tensorflow.Operations return _op.outputs[0]; } - public static (Tensor, Tensor, Tensor) _fused_batch_norm(Tensor x, + public static Tensor bias_add_grad(Tensor out_backprop, + string data_format = "NHWC", + string name = null) + { + if (data_format == null) + data_format = "NHWC"; + + var _op = _op_def_lib._apply_op_helper("BiasAddGrad", name: name, args: new + { + out_backprop, + data_format + }); + + return _op.outputs[0]; + } + + public static Tensor[] _fused_batch_norm(Tensor x, Tensor scale, Tensor offset, Tensor mean, @@ -75,7 +91,87 @@ namespace Tensorflow.Operations is_training }); - return (_op.outputs[0], _op.outputs[1], _op.outputs[2]); + return _op.outputs; + } + + public static Tensor log_softmax(Tensor logits, string name = null) + { + var _op = _op_def_lib._apply_op_helper("LogSoftmax", name: name, args: new + { + logits + }); + + return _op.outputs[0]; + } + + public static Tensor max_pool(Tensor input, + int[] ksize, + int[] strides, + string padding, + string data_format = "NHWC", + string name = null) + { + var _op = _op_def_lib._apply_op_helper("MaxPool", name: name, args: new + { + input, + ksize, + strides, + padding, + data_format, + }); + + return _op.outputs[0]; + } + + public static Tensor[] top_kv2(Tensor input, int k, bool sorted = true, string name = null) + { + var _op = _op_def_lib._apply_op_helper("TopKV2", name: name, args: new + { + input, + k, + sorted + }); + + return _op.outputs; + } + + public static Tensor relu_grad(Tensor gradients, Tensor features, string name = null) + { + var _op = _op_def_lib._apply_op_helper("ReluGrad", name: name, args: new + { + gradients, + features + }); + + return _op.outputs[0]; + } + + public static Tensor softmax(Tensor logits, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Softmax", name: name, args: new + { + logits + }); + + return _op.outputs[0]; + } + + /// + /// Computes softmax cross entropy cost and gradients to backpropagate. + /// + /// + /// + /// + /// + public static (Tensor, Tensor) softmax_cross_entropy_with_logits(Tensor features, Tensor labels, string name = null) + { + var _op = _op_def_lib._apply_op_helper("SoftmaxCrossEntropyWithLogits", name: name, args: new + { + features, + labels + }); + + return (_op.outputs[0], _op.outputs[1]); } } } diff --git a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs index a3535838..3f4b3545 100644 --- a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs +++ b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs @@ -42,7 +42,7 @@ namespace Tensorflow var attrs = new Dictionary(); var inputs = new List(); var input_types = new List(); - dynamic values = null; + object values = null; return with(ops.name_scope(name), scope => { @@ -116,7 +116,7 @@ namespace Tensorflow else if (default_type_attr_map.ContainsKey(input_arg.TypeAttr)) default_dtype = (DataType)default_type_attr_map[input_arg.TypeAttr]; - values = ops.internal_convert_to_tensor(values, + var value = ops.internal_convert_to_tensor(values, name: input_name, dtype: dtype.as_tf_dtype(), as_ref: input_arg.IsRef, @@ -125,7 +125,7 @@ namespace Tensorflow //if (!String.IsNullOrEmpty(input_arg.TypeAttr)) //attrs[input_arg.TypeAttr] = values.dtype; - values = new Tensor[] { values }; + values = new Tensor[] { value }; } if (values is Tensor[] values2) diff --git a/src/TensorFlowNET.Core/Operations/Operation.Control.cs b/src/TensorFlowNET.Core/Operations/Operation.Control.cs index 74078e27..39a011a8 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.Control.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.Control.cs @@ -7,7 +7,7 @@ namespace Tensorflow { public partial class Operation { - private CondContext _control_flow_context; + private IControlFlowContext _control_flow_context; /// /// Add this op to its control flow context. @@ -18,19 +18,30 @@ namespace Tensorflow { } + + if (_control_flow_context != null) + _control_flow_context.AddOp(this); + } + + public void _add_control_input(Operation op) + { + c_api.TF_AddControlInput(_handle, op); } public void _add_control_inputs(Operation[] ops) { - foreach(var op in ops) - { - c_api.TF_AddControlInput(graph, op); - } + foreach (var op in ops) + _add_control_input(op); } - public void _set_control_flow_context(CondContext ctx) + public void _set_control_flow_context(IControlFlowContext ctx) { _control_flow_context = ctx; } + + public IControlFlowContext _get_control_flow_context() + { + return _control_flow_context; + } } } diff --git a/src/TensorFlowNET.Core/Operations/Operation.Output.cs b/src/TensorFlowNET.Core/Operations/Operation.Output.cs index 3ec16704..5b0b43b3 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.Output.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.Output.cs @@ -1,4 +1,5 @@ -using System; +//using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs index d6fb63c1..0979c150 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.cs @@ -1,4 +1,5 @@ using Google.Protobuf.Collections; +//using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; @@ -102,8 +103,12 @@ namespace Tensorflow } } + // Dict mapping op name to file and line information for op colocation + // context managers. + _control_flow_context = graph._get_control_flow_context(); + // This will be set by self.inputs. - if(op_def == null) + if (op_def == null) op_def = g.GetOpDef(node_def.Op); var grouped_inputs = _reconstruct_sequence_inputs(op_def, inputs, node_def.Attr); @@ -185,7 +190,10 @@ namespace Tensorflow if (oneof_value == "type") return x.Type; - return x.GetType().GetProperty(oneof_value).GetValue(x); + object result = x.GetType().GetProperty(oneof_value).GetValue(x); + if (result is Google.Protobuf.ByteString byteString) + return byteString.ToStringUtf8(); + return result; } public TF_AttrMetadata GetAttributeMetadata(string attr_name, Status s) diff --git a/src/TensorFlowNET.Core/Operations/array_ops.py.cs b/src/TensorFlowNET.Core/Operations/array_ops.py.cs index 83acd21e..753b3103 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.py.cs @@ -7,7 +7,8 @@ namespace Tensorflow { public class array_ops : Python { - public static Tensor placeholder_with_default(T input, int[] shape, string name = null) => gen_array_ops.placeholder_with_default(input, shape, name); + public static Tensor placeholder_with_default(T input, int[] shape, string name = null) + => gen_array_ops.placeholder_with_default(input, shape, name); public static Tensor zeros(Shape shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) { @@ -41,20 +42,85 @@ namespace Tensorflow else { tShape = constant_op._tensor_shape_tensor_conversion_function(shape.as_shape()); - var c = constant_op.constant(0); + var c = constant_op.constant(0, dtype: dtype); return gen_array_ops.fill(tShape, c, name: name); } } - public static Tensor expand_dims(Tensor input, int axis = -1, string name = null, int dim = -1) => expand_dims_v2(input, axis, name); + public static Tensor _autopacking_conversion_function(object[] v, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool as_ref = false) + { + var inferred_dtype = _get_dtype_from_nested_lists(v); + if (dtype == TF_DataType.DtInvalid) + dtype = inferred_dtype; - private static Tensor expand_dims_v2(Tensor input, int axis, string name = null) => gen_array_ops.expand_dims(input, axis, name); + return _autopacking_helper(v, dtype, name == null ? "packed" : name); + } - public static Tensor rank(Tensor input, string name = null) + private static TF_DataType _get_dtype_from_nested_lists(object[] list_or_tuple) { - return math_ops.rank_internal(input, name, optimize: true); + TF_DataType dtype = TF_DataType.DtInvalid; + + foreach(var obj in list_or_tuple) + { + switch (obj) + { + case Tensor t: + dtype = t.dtype.as_base_dtype(); + break; + } + + if (dtype != TF_DataType.DtInvalid) + break; + } + + return dtype; } + public static Tensor _autopacking_helper(object[] list_or_tuple, TF_DataType dtype, string name) + { + var must_pack = false; + var converted_elems = new List(); + return with(ops.name_scope(name), scope => + { + foreach (var (i, elem) in enumerate(list_or_tuple)) + { + converted_elems.Add(elem); + must_pack = true; + } + + if(must_pack) + { + var elems_as_tensors = new List(); + foreach (var (i, elem) in enumerate(converted_elems)) + { + if (elem is Tensor tensor) + elems_as_tensors.Add(tensor); + else + { + var elem_tensor = constant_op.constant(elem, dtype: dtype, name: i.ToString()); + elems_as_tensors.Add(elem_tensor); + } + } + + return gen_array_ops.pack(elems_as_tensors.ToArray(), name: scope); + } + else + { + // return converted_elems.ToArray(); + throw new NotImplementedException("_autopacking_helper.converted_elems"); + } + }); + } + + public static Tensor expand_dims(Tensor input, int axis = -1, string name = null, int dim = -1) + => expand_dims_v2(input, axis, name); + + private static Tensor expand_dims_v2(Tensor input, int axis, string name = null) + => gen_array_ops.expand_dims(input, axis, name); + + public static Tensor rank(Tensor input, string name = null) + => math_ops.rank_internal(input, name, optimize: true); + /// /// Creates a tensor with all elements set to 1. /// @@ -66,10 +132,8 @@ namespace Tensorflow public static Tensor ones_like(T tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true) => ones_like_impl(tensor, dtype, name, optimize); - public static Tensor reshape(Tensor tensor, Tensor shape, string name = null) - { - return gen_array_ops.reshape(tensor, shape, null); - } + public static Tensor reshape(T1 tensor, T2 shape, string name = null) + => gen_array_ops.reshape(tensor, shape, null); private static Tensor ones_like_impl(T tensor, TF_DataType dtype, string name, bool optimize = true) { @@ -97,6 +161,18 @@ namespace Tensorflow }); } + public static Tensor ones(Tensor[] shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) + { + dtype = dtype.as_base_dtype(); + return with(ops.name_scope(name, "ones", new { shape }), scope => + { + name = scope; + var shape1 = ops.convert_to_tensor(shape, dtype: TF_DataType.TF_INT32); + var output = gen_array_ops.fill(shape1, constant_op.constant(1, dtype: dtype), name: name); + return output; + }); + } + public static Tensor ones(int[] dims, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) { dtype = dtype.as_base_dtype(); @@ -109,6 +185,44 @@ namespace Tensorflow }); } + public static Tensor one_hot(Tensor indices, int depth, + Tensor on_value = null, + Tensor off_value = null, + TF_DataType dtype = TF_DataType.DtInvalid, + int axis = -1, + string name = null) + { + return with(ops.name_scope(name, "one_hot", new { indices, depth, dtype }), scope => + { + name = scope; + var on_exists = false; + var off_exists = false; + var on_dtype = TF_DataType.DtInvalid; + var off_dtype = TF_DataType.DtInvalid; + + if (dtype == TF_DataType.DtInvalid) + dtype = TF_DataType.TF_FLOAT; + + if(!on_exists) + { + on_value = ops.convert_to_tensor(1, dtype, name: "on_value"); + on_dtype = dtype; + } + + if (!off_exists) + { + off_value = ops.convert_to_tensor(0, dtype, name = "off_value"); + off_dtype = dtype; + } + + return gen_array_ops.one_hot(indices, depth, + on_value: on_value, + off_value: off_value, + axis: axis, + name: name); + }); + } + public static Tensor where(Tensor condition, Tensor x = null, Tensor y = null, string name = null) { if( x == null && y == null) @@ -136,14 +250,10 @@ namespace Tensorflow /// /// A `Tensor` of type `out_type`. public static Tensor shape(Tensor input, string name = null, TF_DataType out_type = TF_DataType.TF_INT32) - { - return shape_internal(input, name, optimize: true, out_type: out_type); - } + => shape_internal(input, name, optimize: true, out_type: out_type); public static Tensor size(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32) - { - return size_internal(input, name, optimize: optimize, out_type: out_type); - } + => size_internal(input, name, optimize: optimize, out_type: out_type); private static Tensor shape_internal(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32) { @@ -168,32 +278,21 @@ namespace Tensorflow private static Tensor size_internal(Tensor input, string name = null, bool optimize = true, TF_DataType out_type = TF_DataType.TF_INT32) { - return with(ops.name_scope(name, "Size", new Tensor[] { input }), scope => + return with(ops.name_scope(name, "Size", new { input }), scope => { name = scope; - if (!tf.context.executing_eagerly()) + var input_tensor = ops.convert_to_tensor(input); + var input_shape = tensor_util.to_shape(input_tensor.shape); + if (optimize) { - var input_tensor = ops.convert_to_tensor(input); - var input_shape = tensor_util.to_shape(input_tensor.shape); - if (optimize) + if (input_shape.is_fully_defined()) { - if (input_shape.is_fully_defined()) - { - var nd = np.array(input_tensor.shape, out_type.as_numpy_datatype()); - return constant_op.constant(nd, name: name); - } + return constant_op.constant(input_shape.Size, dtype: out_type, name: name); } - - return gen_array_ops.size(input, name: name, out_type: out_type); - } - else - { - // result = gen_array_ops.shape(); - throw new NotImplementedException("array_ops.size_internal"); } - return null; + return gen_array_ops.size(input, name: name, out_type: out_type); }); } @@ -234,8 +333,46 @@ namespace Tensorflow /// /// public static Tensor stop_gradient(Tensor input, string name = null) + => gen_array_ops.stop_gradient(input, name); + + /// + /// Extracts a strided slice of a tensor (generalized python array indexing). + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor strided_slice(Tensor input_, Tensor begin, Tensor end, + Tensor strides = null, + int begin_mask = 0, + int end_mask = 0, + int ellipsis_mask = 0, + int new_axis_mask = 0, + int shrink_axis_mask = 0, + string name = null) { - return gen_array_ops.stop_gradient(input, name); + var op = gen_array_ops.strided_slice( + input: input_, + begin: begin, + end: end, + strides: strides, + begin_mask: begin_mask, + end_mask: end_mask, + ellipsis_mask: ellipsis_mask, + new_axis_mask: new_axis_mask, + shrink_axis_mask: shrink_axis_mask, + name: name); + + string parent_name = name; + + return op; } /// @@ -256,14 +393,14 @@ namespace Tensorflow /// Contains the same data as `input`, but has one or more dimensions of /// size 1 removed. public static Tensor squeeze(Tensor input, int[] axis = null, string name = null, int[] squeeze_dims = null) - { - return gen_array_ops.squeeze(input, axis, name); - } + => gen_array_ops.squeeze(input, axis, name); public static Tensor identity(Tensor input, string name = null) - { - return gen_array_ops.identity(input, name); - } + => gen_array_ops.identity(input, name); + + public static Tensor invert_permutation(Tensor x, string name = null) + => gen_array_ops.invert_permutation(x, name: name); + /// /// Computes the shape of a broadcast given symbolic shapes. /// When shape_x and shape_y are Tensors representing shapes(i.e.the result of @@ -279,27 +416,33 @@ namespace Tensorflow /// A rank 1 integer `Tensor`, representing the shape of y. /// A rank 1 integer `Tensor` representing the broadcasted shape. public static Tensor broadcast_dynamic_shape(Tensor shape_x, Tensor shape_y) - { - return gen_array_ops.broadcast_args(shape_x, shape_y); - } + => gen_array_ops.broadcast_args(shape_x, shape_y); public static Tensor broadcast_static_shape(Tensor shape_x, Tensor shape_y) - { - return Framework.common_shapes.broadcast_shape(shape_x, shape_y); - } + => Framework.common_shapes.broadcast_shape(shape_x, shape_y); public static Tensor gather(Tensor @params, Tensor indices, string name = null, int axis = 0) - { - return gen_array_ops.gather_v2(@params, indices, axis, name: name); - } + => gen_array_ops.gather_v2(@params, indices, axis, name: name); - public static Tensor transpose(Tensor a, int[] perm = null, string name = "transpose", bool conjugate = false) + public static Tensor transpose(T1 a, T2 perm, string name = "transpose", bool conjugate = false) { return with(ops.name_scope(name, "transpose", new { a }), scope => { - name = scope; - return gen_array_ops.transpose(a, perm, name); + return gen_array_ops.transpose(a, perm, name: scope); }); } + + public static Tensor slice(Tensor input, Tb[] begin, Ts[] size, string name = null) + => gen_array_ops.slice(input, begin, size, name: name); + + public static Tensor stack(object values, int axis = 0, string name = "stack") + { + if (axis == 0) + // If the input is a constant list, it can be converted to a constant op + return ops.convert_to_tensor(values, name: name); + + throw new NotImplementedException("array_ops.stack"); + } + } } diff --git a/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs b/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs index bca74989..03773c9f 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using Tensorflow.Operations; +using util = Tensorflow.control_flow_util; namespace Tensorflow { @@ -137,9 +138,25 @@ namespace Tensorflow return gen_array_ops.identity(data, name: name); } - public static (Tensor, Tensor) cond(Tensor pred, - Func<(Tensor, Tensor, Tensor)> true_fn = null, - Func<(Tensor, Tensor, Tensor)> false_fn = null, + /// + /// Forwards `data` to an output determined by `pred`. + /// + /// + /// + /// + /// + public static (Tensor, Tensor) _SwitchRefOrTensor(Tensor data, Tensor pred, string name = "Switch") + { + data = ops.convert_to_tensor_or_indexed_slices(data, name: "data"); + + ops.colocate_with(data, ignore_existing: true); + + return @switch(data, pred, name: name); + } + + public static Tensor[] cond(Tensor pred, + Func true_fn = null, + Func false_fn = null, bool strict = false, string name = null) { @@ -158,20 +175,46 @@ namespace Tensorflow // Build the graph for the true branch in a new context. var context_t = new CondContext(pred, pivot_1, branch: 1); context_t.Enter(); - var res_t = context_t.BuildCondBranch(true_fn); + var (orig_res_t, res_t) = context_t.BuildCondBranch(true_fn); context_t.Exit(); // Build the graph for the false branch in a new context. var context_f = new CondContext(pred, pivot_2, branch: 0); context_f.Enter(); - var res_f = context_f.BuildCondBranch(false_fn); + var (orig_res_f, res_f) = context_f.BuildCondBranch(false_fn); context_f.Exit(); - var res_t_flat = new Tensor[] { res_t.Item1, res_t.Item2, res_t.Item3 }; - var res_f_flat = new Tensor[] { res_f.Item1, res_f.Item2, res_f.Item3 }; + var res_t_flat = res_t; + var res_f_flat = res_f; + + var merges = zip(res_f_flat, res_t_flat) + .Select(pair => merge(new Tensor[] { pair.Item1, pair.Item2 })) + .ToArray(); + + merges = _convert_flows_to_tensorarrays(orig_res_t, merges); + + ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_t); + ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_f); + return merges; + }); + } - return (p_2, p_1); + public static Tensor[] _convert_flows_to_tensorarrays(T[] tensors_or_tensorarrays, Tensor[] tensors_or_flows) + { + // zip(tensors_or_tensorarrays, tensors_or_flows).Select((ta, t_or_flow) => ta).ToArray(); + return tensors_or_flows; + } + + public static Tensor merge(Tensor[] inputs, string name = null) + { + return with(ops.name_scope(name, "Merge", inputs), scope => + { + name = scope; + inputs = inputs.Select(inp => + ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref: true)) + .ToArray(); + return gen_control_flow_ops.merge(inputs, name).Item1; }); } @@ -200,5 +243,18 @@ namespace Tensorflow return gen_control_flow_ops.@switch(data, pred, name: name); }); } + + public static Tensor ZerosLikeOutsideLoop(Operation op, int index) + { + var val = op.outputs[index]; + if (!util.IsSwitch(op)) + { + if (val.dtype == TF_DataType.TF_RESOURCE) + throw new NotImplementedException("ZerosLikeOutsideLoop"); + return array_ops.zeros_like(val, optimize: false); + } + + throw new NotImplementedException("ZerosLikeOutsideLoop"); + } } } diff --git a/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs b/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs index 4654261e..0de8bdeb 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Operations; namespace Tensorflow { @@ -15,5 +16,22 @@ namespace Tensorflow { return op.type == "Exit" || op.type == "RefExit"; } + + /// + /// Return true if `op` is a Switch. + /// + /// + /// + public static bool IsSwitch(Operation op) + { + return op.type == "Switch" || op.type == "RefSwitch"; + } + + public static IControlFlowContext GetOutputContext(Operation op) + { + var ctxt = op._get_control_flow_context(); + + return ctxt; + } } } diff --git a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs index f2c5841f..3f168397 100644 --- a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs @@ -27,16 +27,9 @@ namespace Tensorflow return _op.outputs[0]; } - public static Tensor greater(Tx x, Ty y, string name = null) + public static Tensor pack(Tensor[] values, int axis = 0, string name = null) { - var _op = _op_def_lib._apply_op_helper("Greater", name: name, args: new { x, y }); - - return _op.outputs[0]; - } - - public static Tensor less(Tx x, Ty y, string name = null) - { - var _op = _op_def_lib._apply_op_helper("Less", name: name, args: new { x, y }); + var _op = _op_def_lib._apply_op_helper("Pack", name: name, args: new { values, axis }); return _op.outputs[0]; } @@ -68,6 +61,13 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor invert_permutation(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("InvertPermutation", name, new { x }); + + return _op.outputs[0]; + } + public static Tensor log(Tensor x, string name = null) { var _op = _op_def_lib._apply_op_helper("Log", name: name, args: new { x }); @@ -110,7 +110,13 @@ namespace Tensorflow return (_op.outputs[0], _op.outputs[1]); } - public static Tensor reshape(Tensor tensor, Tensor shape, string name = null) + public static Tensor reshape(T1 tensor, T2 shape, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Reshape", name, new { tensor, shape }); + return _op.outputs[0]; + } + + public static Tensor reshape(Tensor tensor, int[] shape, string name = null) { var _op = _op_def_lib._apply_op_helper("Reshape", name, new { tensor, shape }); return _op.outputs[0]; @@ -121,6 +127,17 @@ namespace Tensorflow throw new NotImplementedException("where"); } + public static Tensor one_hot(Tensor indices, int depth, + Tensor on_value = null, + Tensor off_value = null, + TF_DataType dtype = TF_DataType.DtInvalid, + int axis = -1, + string name = null) + { + var _op = _op_def_lib._apply_op_helper("OneHot", name, new { indices, depth, on_value, off_value, axis }); + return _op.outputs[0]; + } + /// /// A placeholder op that passes through `input` when its output is not fed. /// @@ -140,6 +157,12 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor scatter_nd(Tensor indices, Tensor updates, Tensor[] shape, string name = null) + { + var _op = _op_def_lib._apply_op_helper("ScatterNd", name, new { indices, updates, shape }); + return _op.outputs[0]; + } + public static Tensor shape(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string name = null) { var _op = _op_def_lib._apply_op_helper("Shape", name, new { input, out_type }); @@ -163,7 +186,7 @@ namespace Tensorflow return _op.outputs[0]; } - public static Tensor transpose(Tensor x, int[] perm, string name = null) + public static Tensor transpose(T1 x, T2 perm, string name = null) { var _op = _op_def_lib._apply_op_helper("Transpose", name, new { x, perm }); return _op.outputs[0]; @@ -174,12 +197,44 @@ namespace Tensorflow var _op = _op_def_lib._apply_op_helper("ZerosLike", name, new { x }); return _op.outputs[0]; } + public static Tensor stop_gradient(Tensor x, string name = null) { var _op = _op_def_lib._apply_op_helper("StopGradient", name, args: new { input = x, name }); return _op.outputs[0]; } + + public static Tensor strided_slice(Tensor input, Tensor begin, Tensor end, Tensor strides, + int begin_mask = 0, + int end_mask = 0, + int ellipsis_mask = 0, + int new_axis_mask = 0, + int shrink_axis_mask = 0, + string name = null) + { + var _op = _op_def_lib._apply_op_helper("StridedSlice", name, new + { + input, + begin, + end, + strides, + begin_mask, + end_mask, + ellipsis_mask, + new_axis_mask, + shrink_axis_mask + }); + + return _op.outputs[0]; + } + + public static Tensor slice(Tensor input, Tb[] begin, Ts[] size, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Slice", name, new { input, begin, size }); + return _op.outputs[0]; + } + /// /// Removes dimensions of size 1 from the shape of a tensor. /// Given a tensor `input`, this operation returns a tensor of the same type with diff --git a/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs b/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs index faedfae4..21447c57 100644 --- a/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs @@ -4,7 +4,7 @@ using System.Text; namespace Tensorflow { - public class gen_control_flow_ops + public class gen_control_flow_ops : Python { public static OpDefLibrary _op_def_lib = new OpDefLibrary(); @@ -21,5 +21,12 @@ namespace Tensorflow return (_op.outputs[0], _op.outputs[1]); } + + public static (Tensor, Tensor) merge(Tensor[] inputs, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Merge", name, new { inputs }); + + return (_op.outputs[0], _op.outputs[1]); + } } } diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index 9d1e8788..a3f017bf 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -9,6 +9,30 @@ namespace Tensorflow public static class gen_math_ops { public static OpDefLibrary _op_def_lib = new OpDefLibrary(); + + /// + /// Returns the index with the largest value across dimensions of a tensor. + /// + /// + /// + /// + /// + /// + public static Tensor arg_max(Tensor input, int dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) + => _op_def_lib._apply_op_helper("ArgMax", name, args: new { input, dimension, output_type }).outputs[0]; + + /// + /// Returns the index with the smallest value across dimensions of a tensor. + /// + /// + /// + /// + /// + /// + public static Tensor arg_min(Tensor input, int dimension, TF_DataType output_type= TF_DataType.TF_INT64, string name= null) + =>_op_def_lib._apply_op_helper("ArgMin", name, args: new { input, dimension, output_type }).outputs[0]; + + /// /// Computes the mean of elements across dimensions of a tensor. /// Reduces `input` along the dimensions given in `axis`. Unless /// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in /// `axis`. If `keep_dims` is true, the reduced dimensions are retained with length 1. @@ -20,16 +44,30 @@ namespace Tensorflow /// An optional `bool`. Defaults to `False`. If true, retain reduced dimensions with length 1. /// A name for the operation (optional). /// A `Tensor`. Has the same type as `input`. - public static Tensor mean(Tensor input, Tensor axis, bool keep_dims= false, string name = null) + public static Tensor mean(T1 input, T2 axis, bool keep_dims= false, string name = null) { var _op = _op_def_lib._apply_op_helper("Mean", name, args: new { input, reduction_indices = axis, keep_dims = keep_dims }); return _op.outputs[0]; } - public static Tensor mean(Tensor input, int[] axis, bool keep_dims = false, string name = null) + public static Tensor prod(T1 input, T2 axis, bool keep_dims = false, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Prod", name, args: new { input, reduction_indices = axis, keep_dims }); + + return _op.outputs[0]; + } + + public static Tensor acos(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Acos", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor asin(Tensor x, string name = null) { - var _op = _op_def_lib._apply_op_helper("Mean", name, args: new { input, reduction_indices = axis, keep_dims = keep_dims, name }); + var _op = _op_def_lib._apply_op_helper("Asin", name, args: new { x }); return _op.outputs[0]; } @@ -41,6 +79,83 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor atan(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Atan", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor ceil(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Ceil", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor cos(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Cos", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor cosh(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Cosh", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor floor(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Floor", name, args: new { x }); + + return _op.outputs[0]; + } + + public static Tensor _clip_by_value(Tensor t, Tensor clip_value_min, Tensor clip_value_max, string name = null) + { + var _op = _op_def_lib._apply_op_helper("ClipByValue", name, args: new { t, clip_value_min, clip_value_max }); + + return _op.outputs[0]; + } + + public static Tensor greater(Tx x, Ty y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Greater", name: name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor greater_equal(Tx x, Ty y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("GreaterEqual", name: name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor less(Tx x, Ty y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Less", name: name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor less_equal(Tx x, Ty y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("LessEqual", name: name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor log1p(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Log1p", name, args: new { x }); + + return _op.outputs[0]; + } + public static Tensor squared_difference(Tensor x, Tensor y, string name = null) { var _op = _op_def_lib._apply_op_helper("SquaredDifference", name, args: new { x, y, name }); @@ -128,6 +243,27 @@ namespace Tensorflow return _op.outputs[0]; } + /// + /// Returns the truth value of (x == y) element-wise. + /// + /// + /// + /// + /// + public static Tensor equal(Tensor x, Tensor y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Equal", name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor atan2(Tensor y, Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Atan2", name, args: new { y, x }); + + return _op.outputs[0]; + } + public static Tensor mul(Tensor x, Tensor y, string name = null) { var _op = _op_def_lib._apply_op_helper("Mul", name, args: new { x, y }); @@ -142,6 +278,13 @@ namespace Tensorflow return _op.outputs[0]; } + public static Tensor reciprocal(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Reciprocal", name, args: new { x }); + + return _op.outputs[0]; + } + public static Tensor floor_mod(Tensor x, Tensor y, string name = null) { var _op = _op_def_lib._apply_op_helper("FloorMod", name, args: new { x, y }); @@ -186,13 +329,34 @@ namespace Tensorflow return _op.outputs[0]; } - public static Tensor _max(Tensor input, int[] axis, bool keep_dims=false, string name = null) + public static Tensor minimum(T1 x, T2 y, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Minimum", name, args: new { x, y }); + + return _op.outputs[0]; + } + + public static Tensor _abs(Tensor x, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Abs", name, new { x }); + + return _op.outputs[0]; + } + + public static Tensor _max(Tx input, Ty axis, bool keep_dims=false, string name = null) { var _op = _op_def_lib._apply_op_helper("Max", name, new { input, reduction_indices = axis, keep_dims }); return _op.outputs[0]; } + public static Tensor _min(Tx input, Ty axis, bool keep_dims = false, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Min", name, new { input, reduction_indices = axis, keep_dims }); + + return _op.outputs[0]; + } + public static Tensor pow(Tx x, Ty y, string name = null) { var _op = _op_def_lib._apply_op_helper("Pow", name, args: new { x, y }); @@ -200,7 +364,14 @@ namespace Tensorflow return _op.outputs[0]; } - public static Tensor sum(Tensor input, Tensor axis = null, bool keep_dims = false, string name = null) + public static Tensor _sum(Tensor input, Tensor axis = null, bool keep_dims = false, string name = null) + { + var _op = _op_def_lib._apply_op_helper("Sum", name, args: new { input, reduction_indices = axis, keep_dims }); + + return _op.outputs[0]; + } + + public static Tensor _sum(Tensor input, int axis, bool keep_dims = false, string name = null) { var _op = _op_def_lib._apply_op_helper("Sum", name, args: new { input, reduction_indices = axis, keep_dims }); diff --git a/src/TensorFlowNET.Core/Operations/math_ops.py.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs similarity index 70% rename from src/TensorFlowNET.Core/Operations/math_ops.py.cs rename to src/TensorFlowNET.Core/Operations/math_ops.cs index e35a094b..8077a87a 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -2,12 +2,40 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Framework; namespace Tensorflow { + /// + /// python\ops\math_ops.py + /// public class math_ops : Python { - public static Tensor add(Tensor x, Tensor y, string name = null) => gen_math_ops.add(x, y, name); + public static Tensor abs(Tensor x, string name = null) + { + return with(ops.name_scope(name, "Abs", new { x }), scope => + { + x = ops.convert_to_tensor(x, name: "x"); + if (x.dtype.is_complex()) + throw new NotImplementedException("math_ops.abs for dtype.is_complex"); + //return gen_math_ops.complex_abs(x, Tout: x.dtype.real_dtype, name: name); + return gen_math_ops._abs(x, name: name); + }); + } + + public static Tensor add(Tensor x, Tensor y, string name = null) + => gen_math_ops.add(x, y, name); + + public static Tensor add(Tensor x, string name = null) + { + return with(ops.name_scope(name, "Abs", new { x }), scope => + { + name = scope; + x = ops.convert_to_tensor(x, name: "x"); + + return gen_math_ops._abs(x, name: name); + }); + } public static Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) { @@ -17,6 +45,7 @@ namespace Tensorflow return with(ops.name_scope(name, "Cast", new { x }), scope => { + name = scope; x = ops.convert_to_tensor(x, name: "x"); if (x.dtype.as_base_dtype() != base_type) x = gen_math_ops.cast(x, base_type, name: name); @@ -36,12 +65,44 @@ namespace Tensorflow /// dimensions.Must be in the range `[-rank(input_tensor), rank(input_tensor))`. /// If true, retains reduced dimensions with length 1. /// A name for the operation (optional). - public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null) + public static Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null, int? reduction_indices = null) { var r = _ReductionDims(input_tensor, axis); - var m = gen_math_ops.mean(input_tensor, (int[]) r, keepdims, name); - return _may_reduce_to_scalar(keepdims,axis, m); + if (axis == null) + { + var m = gen_math_ops.mean(input_tensor, r, keepdims, name); + return _may_reduce_to_scalar(keepdims, axis, m); + } + else + { + var m = gen_math_ops.mean(input_tensor, axis, keepdims, name); + return _may_reduce_to_scalar(keepdims, axis, m); + } + } + + /// + /// Computes the product of elements across dimensions of a tensor. + /// + /// + /// + /// + /// + /// + public static Tensor reduce_prod(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null) + { + var r = _ReductionDims(input_tensor, axis); + if (axis == null) + { + var m = gen_math_ops.prod(input_tensor, r, keepdims, name); + return _may_reduce_to_scalar(keepdims, axis, m); + } + else + { + var m = gen_math_ops.prod(input_tensor, axis, keepdims, name); + return _may_reduce_to_scalar(keepdims, axis, m); + } } + /// /// Returns (x - y)(x - y) element-wise. /// @@ -60,6 +121,11 @@ namespace Tensorflow return gen_math_ops.square(x, name); } + public static Tensor subtract(Tx x, Ty y, string name = null) + { + return gen_math_ops.sub(x, y, name); + } + public static Tensor log(Tensor x, string name = null) { return gen_math_ops.log(x, name); @@ -87,6 +153,16 @@ namespace Tensorflow return gen_data_flow_ops.dynamic_stitch(a1, a2); } + /// + /// Computes the reciprocal of x element-wise. + /// + /// + /// + /// + public static Tensor reciprocal(Tensor x, string name = null) + => gen_math_ops.reciprocal(x, name: name); + + /// /// Computes log(sum(exp(elements across dimensions of a tensor))). /// Reduces `input_tensor` along the dimensions given in `axis`. @@ -129,7 +205,10 @@ namespace Tensorflow public static Tensor reduce_max(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null) { - return _may_reduce_to_scalar(keepdims, axis, gen_math_ops._max(input_tensor, (int[])_ReductionDims(input_tensor, axis), keepdims, name)); + var r = _ReductionDims(input_tensor, axis); + var max = (axis != null) ? gen_math_ops._max(input_tensor, axis, keepdims, name) : + gen_math_ops._max(input_tensor, r, keepdims, name); + return _may_reduce_to_scalar(keepdims, axis, max); } /// @@ -160,20 +239,29 @@ namespace Tensorflow throw new NotImplementedException(); } - public static Tensor reduce_sum(Tensor input_tensor, Tensor axis = null, bool keepdims = false) + public static Tensor reduce_sum(Tensor input_tensor, Tensor axis = null, bool keepdims = false, string name = null) { var r = _ReductionDims(input_tensor, axis); - var m = gen_math_ops.sum(input_tensor, r); - return _may_reduce_to_scalar(keepdims, m); + var m = gen_math_ops._sum(input_tensor, r, keep_dims: keepdims, name: name); + return _may_reduce_to_scalar(keepdims, axis, m); } - private static Tensor _may_reduce_to_scalar(bool keepdims, Tensor output) + public static Tensor reduce_sum(Tensor input_tensor, int axis, bool keepdims = false, string name = null) { - output.shape = new long[0]; + var m = gen_math_ops._sum(input_tensor, axis, keep_dims: keepdims, name: name); + return _may_reduce_to_scalar(keepdims, new int[] { axis }, m); + } + + private static Tensor _may_reduce_to_scalar(bool keepdims, Tensor axis, Tensor output) + { + if (!common_shapes.has_fully_defined_shape(output) && + !keepdims && + axis == null) + output.shape = new long[0]; return output; } - private static Tensor _may_reduce_to_scalar(bool keepdims, int[] axos, Tensor output) + private static Tensor _may_reduce_to_scalar(bool keepdims, int[] axis, Tensor output) { output.shape = new long[0]; return output; @@ -191,19 +279,20 @@ namespace Tensorflow return range(0, rank, 1); } } - - private static object _ReductionDims(Tensor x, int[] axis) + + private static Tensor _ReductionDims(Tensor x, int[] axis) { if (axis != null) { - return axis; + // should return axis. or check before. + return null; } else { - var rank = array_ops.rank(x); + var rank = common_shapes.rank(x); if (rank != null) { - return constant_op.constant(np.arange(rank), TF_DataType.TF_INT32); + return constant_op.constant(np.arange(rank.Value), TF_DataType.TF_INT32); } return range(0, rank, 1); } @@ -221,7 +310,7 @@ namespace Tensorflow if (delta == null) delta = 1; - return with(ops.name_scope(name, "Range", new object[] { start, limit, delta }), scope => + return with(ops.name_scope(name, "Range", new { start, limit, delta }), scope => { name = scope; var start1 = ops.convert_to_tensor(start, name: "start"); @@ -298,5 +387,20 @@ namespace Tensorflow return x; }); } + + public static Tensor truediv(Tensor x, Tensor y, string name = null) + => _truediv_python3(x, y, name); + + public static Tensor _truediv_python3(Tensor x, Tensor y, string name = null) + { + return with(ops.name_scope(name, "truediv", new { x, y }), scope => + { + name = scope; + var x_dtype = x.dtype.as_base_dtype(); + var y_dtype = y.dtype.as_base_dtype(); + + return gen_math_ops.real_div(x, y, name: name); + }); + } } } diff --git a/src/TensorFlowNET.Core/Operations/nn_impl.py.cs b/src/TensorFlowNET.Core/Operations/nn_impl.py.cs index 81515e18..c5de18da 100644 --- a/src/TensorFlowNET.Core/Operations/nn_impl.py.cs +++ b/src/TensorFlowNET.Core/Operations/nn_impl.py.cs @@ -46,7 +46,7 @@ namespace Tensorflow }); } - public static (Tensor, Tensor, Tensor) fused_batch_norm(Tensor x, + public static Tensor[] fused_batch_norm(Tensor x, RefVariable scale, RefVariable offset, Tensor mean, diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs index c7aec377..e54caf66 100644 --- a/src/TensorFlowNET.Core/Operations/nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs @@ -41,5 +41,55 @@ namespace Tensorflow return gen_nn_ops.bias_add(value, bias_tensor, data_format: data_format, name: name); }); } + + public static Tensor log_softmax(Tensor logits, int axis = -1, string name = null) + { + return _softmax(logits, gen_nn_ops.log_softmax, axis, name); + } + + public static Tensor _softmax(Tensor logits, Func compute_op, int dim = -1, string name = null) + { + logits = ops.convert_to_tensor(logits); + + var shape = logits.shape; + bool is_last_dim = dim == -1 || dim == shape.Length - 1; + if (is_last_dim) + return compute_op(logits, name); + + throw new NotImplementedException("_softmax helper"); + } + + public static Tensor softmax_cross_entropy_with_logits_v2_helper(Tensor labels, + Tensor logits, + int axis = -1, + string name = null) + { + return Python.with(ops.name_scope(name, "softmax_cross_entropy_with_logits", new { }), scope => + { + var precise_logits = logits; + var input_rank = array_ops.rank(precise_logits); + var shape = logits.getShape(); + + if (axis != -1) + throw new NotImplementedException("softmax_cross_entropy_with_logits_v2_helper axis != -1"); + + var input_shape = array_ops.shape(precise_logits); + + // Do the actual op computation. + // The second output tensor contains the gradients. We use it in + // _CrossEntropyGrad() in nn_grad but not here. + + var (cost, unused_backprop) = gen_nn_ops.softmax_cross_entropy_with_logits(precise_logits, labels, name: name); + + // The output cost shape should be the input minus axis. + var output_shape = array_ops.slice(input_shape, + new int[] { 0 }, + new Tensor[] { math_ops.subtract(input_rank, 1) }); + + cost = array_ops.reshape(cost, output_shape); + + return cost; + }); + } } } diff --git a/src/TensorFlowNET.Core/Python.cs b/src/TensorFlowNET.Core/Python.cs index 1a7beb72..4888ec88 100644 --- a/src/TensorFlowNET.Core/Python.cs +++ b/src/TensorFlowNET.Core/Python.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Linq; using System.Text; namespace Tensorflow @@ -16,6 +17,11 @@ namespace Tensorflow Console.WriteLine(obj.ToString()); } + protected IEnumerable range(int end) + { + return Enumerable.Range(0, end); + } + public static T New(object args) where T : IPyClass { var instance = Activator.CreateInstance(); @@ -118,14 +124,6 @@ namespace Tensorflow { object obj = propertyDescriptor.GetValue(dyn); string name = propertyDescriptor.Name; - // avoid .net keyword - switch (name) - { - case "_ref_": - name = "ref"; - break; - } - dictionary.Add(name, obj); } return dictionary; diff --git a/src/TensorFlowNET.Core/Sessions/BaseSession.cs b/src/TensorFlowNET.Core/Sessions/BaseSession.cs index 5c84f34a..530a0107 100644 --- a/src/TensorFlowNET.Core/Sessions/BaseSession.cs +++ b/src/TensorFlowNET.Core/Sessions/BaseSession.cs @@ -186,9 +186,10 @@ namespace Tensorflow var result = new NDArray[fetch_list.Length]; for (int i = 0; i < fetch_list.Length; i++) - { result[i] = fetchValue(output_values[i]); - } + + for (int i = 0; i < feed_dict.Length; i++) + feed_dict[i].Value.Dispose(); return result; } @@ -222,6 +223,12 @@ namespace Tensorflow ints[i] = *(int*)(offset + (int)(tensor.itemsize * i)); nd = np.array(ints).reshape(ndims); break; + case TF_DataType.TF_INT64: + var longs = new long[tensor.size]; + for (ulong i = 0; i < tensor.size; i++) + longs[i] = *(long*)(offset + (int)(tensor.itemsize * i)); + nd = np.array(longs).reshape(ndims); + break; case TF_DataType.TF_FLOAT: var floats = new float[tensor.size]; for (ulong i = 0; i < tensor.size; i++) diff --git a/src/TensorFlowNET.Core/Sessions/_ElementFetchMapper.cs b/src/TensorFlowNET.Core/Sessions/_ElementFetchMapper.cs index cec214a4..8cf84cdf 100644 --- a/src/TensorFlowNET.Core/Sessions/_ElementFetchMapper.cs +++ b/src/TensorFlowNET.Core/Sessions/_ElementFetchMapper.cs @@ -10,7 +10,6 @@ namespace Tensorflow /// public class _ElementFetchMapper : _FetchMapper { - private List _unique_fetches = new List(); private Func, object> _contraction_fn; public _ElementFetchMapper(object[] fetches, Func, object> contraction_fn) @@ -32,7 +31,7 @@ namespace Tensorflow /// /// /// - public NDArray build_results(List values) + public override NDArray build_results(List values) { NDArray result = null; @@ -44,6 +43,24 @@ namespace Tensorflow case NDArray value: result = value; break; + case short value: + result = value; + break; + case int value: + result = value; + break; + case long value: + result = value; + break; + case float value: + result = value; + break; + case double value: + result = value; + break; + case string value: + result = value; + break; default: break; } @@ -51,10 +68,5 @@ namespace Tensorflow return result; } - - public List unique_fetches() - { - return _unique_fetches; - } } } diff --git a/src/TensorFlowNET.Core/Sessions/_FetchHandler.cs b/src/TensorFlowNET.Core/Sessions/_FetchHandler.cs index e45e3823..20194c37 100644 --- a/src/TensorFlowNET.Core/Sessions/_FetchHandler.cs +++ b/src/TensorFlowNET.Core/Sessions/_FetchHandler.cs @@ -10,7 +10,7 @@ namespace Tensorflow /// public class _FetchHandler { - private _ElementFetchMapper _fetch_mapper; + private _FetchMapper _fetch_mapper; private List _fetches = new List(); private List _ops = new List(); private List _final_fetches = new List(); @@ -18,7 +18,7 @@ namespace Tensorflow public _FetchHandler(Graph graph, object fetches, Dictionary feeds = null, Action feed_handles = null) { - _fetch_mapper = new _FetchMapper().for_fetch(fetches); + _fetch_mapper = _FetchMapper.for_fetch(fetches); foreach(var fetch in _fetch_mapper.unique_fetches()) { switch (fetch) @@ -58,7 +58,31 @@ namespace Tensorflow { var value = tensor_values[j]; j += 1; - full_values.Add(value); + if (value.ndim == 0) + { + switch (value.dtype.Name) + { + case "Int32": + full_values.Add(value.Data(0)); + break; + case "Int64": + full_values.Add(value.Data(0)); + break; + case "Single": + full_values.Add(value.Data(0)); + break; + case "Double": + full_values.Add(value.Data(0)); + break; + case "String": + full_values.Add(value.Data(0)); + break; + } + } + else + { + full_values.Add(value[np.arange(0, value.shape[0])]); + } } i += 1; } diff --git a/src/TensorFlowNET.Core/Sessions/_FetchMapper.cs b/src/TensorFlowNET.Core/Sessions/_FetchMapper.cs index fc9d6b43..038e9971 100644 --- a/src/TensorFlowNET.Core/Sessions/_FetchMapper.cs +++ b/src/TensorFlowNET.Core/Sessions/_FetchMapper.cs @@ -1,4 +1,5 @@ -using System; +using NumSharp.Core; +using System; using System.Collections.Generic; using System.Text; @@ -6,14 +7,26 @@ namespace Tensorflow { public class _FetchMapper { - public _ElementFetchMapper for_fetch(object fetch) + protected List _unique_fetches = new List(); + + public static _FetchMapper for_fetch(object fetch) { - var fetches = new object[] { fetch }; + var fetches = fetch.GetType().IsArray ? (object[])fetch : new object[] { fetch }; + + if (fetch.GetType().IsArray) + return new _ListFetchMapper(fetches); + else + return new _ElementFetchMapper(fetches, (List fetched_vals) => fetched_vals[0]); + } - return new _ElementFetchMapper(fetches, (List fetched_vals) => - { - return fetched_vals[0]; - }); + public virtual NDArray build_results(List values) + { + return values.ToArray(); + } + + public virtual List unique_fetches() + { + return _unique_fetches; } } } diff --git a/src/TensorFlowNET.Core/Sessions/_ListFetchMapper.cs b/src/TensorFlowNET.Core/Sessions/_ListFetchMapper.cs new file mode 100644 index 00000000..f94a19da --- /dev/null +++ b/src/TensorFlowNET.Core/Sessions/_ListFetchMapper.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tensorflow +{ + public class _ListFetchMapper : _FetchMapper + { + private _FetchMapper[] _mappers; + + public _ListFetchMapper(object[] fetches) + { + _mappers = fetches.Select(fetch => _FetchMapper.for_fetch(fetch)).ToArray(); + _unique_fetches.AddRange(fetches); + } + } +} diff --git a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj index 23d5b0a6..30ce8513 100644 --- a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj +++ b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj @@ -4,23 +4,26 @@ netstandard2.0 TensorFlow.NET Tensorflow - 0.4.2 + 0.6.0 Haiping Chen SciSharp STACK - true + false Apache 2.0 https://github.com/SciSharp/TensorFlow.NET git https://github.com/SciSharp https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 - TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET + TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET, C# Google's TensorFlow binding in .NET Standard. Docs: https://tensorflownet.readthedocs.io - 0.4.2.0 - Added ConfigProto to control CPU and GPU resource. -Fixed import name scope issue. + 0.6.0.0 + Changes since v0.5: +Added K-means Clustering. +Added Nearest Neighbor. +Added a lot of APIs to build neural networks model. +Bug fix. 7.2 - 0.4.2.0 + 0.6.0.0 @@ -44,15 +47,17 @@ Fixed import name scope issue. - + - + + + - + diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs index d31ec927..5998ead3 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs @@ -111,7 +111,7 @@ namespace Tensorflow // Free the original buffer and set flag Deallocator deallocator = (IntPtr values, IntPtr len, ref bool closure) => { - Marshal.FreeHGlobal(dotHandle); + Marshal.FreeHGlobal(values); closure = true; }; diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Operators.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Operators.cs index cd3ac548..6dd5c0ff 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Operators.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Operators.cs @@ -14,6 +14,7 @@ namespace Tensorflow public static Tensor operator -(Tensor x, Tensor y) => BinaryOpWrapper("sub", x, y); public static Tensor operator -(Tensor x, int y) => BinaryOpWrapper("sub", x, y); public static Tensor operator -(Tensor x, double y) => BinaryOpWrapper("sub", x, y); + public static Tensor operator -(float x, Tensor y) => BinaryOpWrapper("Sub", x, y); public static Tensor operator *(float x, Tensor y) => BinaryOpWrapper("mul", x, y); public static Tensor operator *(double x, Tensor y) => BinaryOpWrapper("mul", x, y); @@ -26,12 +27,12 @@ namespace Tensorflow public static Tensor operator %(Tensor x, Tensor y) => BinaryOpWrapper("mod", x, y); - public static Tensor operator >(Tensor x, int y) => gen_array_ops.greater(x, y); - public static Tensor operator >(Tensor x, float y) => gen_array_ops.greater(x, y); - public static Tensor operator >(Tensor x, double y) => gen_array_ops.greater(x, y); - public static Tensor operator <(Tensor x, int y) => gen_array_ops.less(x, y); - public static Tensor operator <(Tensor x, float y) => gen_array_ops.less(x, y); - public static Tensor operator <(Tensor x, double y) => gen_array_ops.less(x, y); + public static Tensor operator >(Tensor x, int y) => gen_math_ops.greater(x, y); + public static Tensor operator >(Tensor x, float y) => gen_math_ops.greater(x, y); + public static Tensor operator >(Tensor x, double y) => gen_math_ops.greater(x, y); + public static Tensor operator <(Tensor x, int y) => gen_math_ops.less(x, y); + public static Tensor operator <(Tensor x, float y) => gen_math_ops.less(x, y); + public static Tensor operator <(Tensor x, double y) => gen_math_ops.less(x, y); private static Tensor BinaryOpWrapper(string name, Tx x, Ty y) { @@ -48,7 +49,7 @@ namespace Tensorflow var x1 = ops.convert_to_tensor(x, dtype: dtype, name: "x"); var y1 = ops.convert_to_tensor(y, dtype: dtype, name: "y"); - switch (name) + switch (name.ToLower()) { case "add": result = gen_math_ops.add(x1, y1, name: scope); diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs index 2b3db534..dffa8ff6 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.cs @@ -1,4 +1,5 @@ -using NumSharp.Core; +//using Newtonsoft.Json; +using NumSharp.Core; using System; using System.Collections.Generic; using System.Linq; @@ -43,6 +44,8 @@ namespace Tensorflow public IntPtr buffer => _handle == IntPtr.Zero ? IntPtr.Zero : c_api.TF_TensorData(_handle); public int num_consumers(TF_Output oper_out) => _handle == IntPtr.Zero ? 0 : c_api.TF_OperationOutputNumConsumers(oper_out); + private TF_Output? _tf_output; + public long[] shape { get @@ -74,7 +77,8 @@ namespace Tensorflow public int[] _shape_tuple() { - return null; + if (shape == null) return null; + return shape.Select(x => (int)x).ToArray(); } public TensorShape getShape() @@ -122,7 +126,10 @@ namespace Tensorflow public TF_Output _as_tf_output() { - return new TF_Output(op, value_index); + if(!_tf_output.HasValue) + _tf_output = new TF_Output(op, value_index); + + return _tf_output.Value; } public T[] Data() @@ -161,7 +168,12 @@ namespace Tensorflow /// A dictionary that maps `Tensor` objects to feed values. /// The `Session` to be used to evaluate this tensor. /// - public NDArray eval(FeedItem[] feed_dict = null, Session session = null) + public NDArray eval(params FeedItem[] feed_dict) + { + return ops._eval_using_default_session(this, feed_dict, graph); + } + + public NDArray eval(Session session, FeedItem[] feed_dict = null) { return ops._eval_using_default_session(this, feed_dict, graph, session); } @@ -186,6 +198,61 @@ namespace Tensorflow } } + public Tensor this[int slice_spec] + { + get + { + var slice_spec_s = new int[] { slice_spec }; + var begin = new List(); + var end = new List(); + var strides = new List(); + + var index = 0; + var (new_axis_mask, shrink_axis_mask) = (0, 0); + var (begin_mask, end_mask) = (0, 0); + var ellipsis_mask = 0; + + foreach(var s in slice_spec_s) + { + { + begin.Add(s); + end.Add(s + 1); + strides.Add(1); + shrink_axis_mask |= (1 << index); + } + + index += 1; + } + + return with(ops.name_scope(null, "strided_slice", new { begin, end, strides }), scope => + { + string name = scope; + if(begin != null) + { + var (packed_begin, packed_end, packed_strides) = + (array_ops.stack(begin.ToArray()), + array_ops.stack(end.ToArray()), + array_ops.stack(strides.ToArray())); + + return gen_array_ops.strided_slice( + this, + packed_begin, + packed_end, + packed_strides, + begin_mask: begin_mask, + end_mask: end_mask, + shrink_axis_mask: shrink_axis_mask, + new_axis_mask: new_axis_mask, + ellipsis_mask: ellipsis_mask, + name: name); + } + + throw new NotImplementedException(""); + }); + } + + } + public override string ToString() { if(NDims == 0) diff --git a/src/TensorFlowNET.Core/Tensors/dtypes.cs b/src/TensorFlowNET.Core/Tensors/dtypes.cs index e010c432..f14affbc 100644 --- a/src/TensorFlowNET.Core/Tensors/dtypes.cs +++ b/src/TensorFlowNET.Core/Tensors/dtypes.cs @@ -31,9 +31,15 @@ namespace Tensorflow switch (type.Name) { + case "Boolean": + dtype = TF_DataType.TF_BOOL; + break; case "Int32": dtype = TF_DataType.TF_INT32; break; + case "Int64": + dtype = TF_DataType.TF_INT64; + break; case "Single": dtype = TF_DataType.TF_FLOAT; break; diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index bee8e68e..73bc6006 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -38,7 +38,8 @@ namespace Tensorflow { return MakeNdarray(tensor.op.get_attr("value") as TensorProto); } - throw new NotImplementedException("_ConstantValue"); + + return null; } public static NDArray MakeNdarray(TensorProto tensor) @@ -50,6 +51,15 @@ namespace Tensorflow if (tensor.TensorContent.Length > 0) return np.frombuffer(tensor.TensorContent.ToByteArray(), tensor_dtype) .reshape(shape); + else if (tensor.Dtype == DataType.DtHalf || tensor.Dtype == DataType.DtBfloat16) + ; + else if (tensor.Dtype == DataType.DtFloat) + ; + else if (new DataType[] { DataType.DtInt32, DataType.DtUint8 }.Contains(tensor.Dtype)) + if (tensor.IntVal.Count == 1) + return np.repeat(np.array(tensor.IntVal[0]), Convert.ToInt32(num_elements)) + .reshape(shape); + throw new NotImplementedException("MakeNdarray"); } @@ -101,6 +111,9 @@ namespace Tensorflow case int intVal: nparray = intVal; break; + case long intVal: + nparray = intVal; + break; case int[] intVals: nparray = np.array(intVals); break; @@ -216,11 +229,15 @@ namespace Tensorflow switch (nparray.dtype.Name) { case "Bool": + case "Boolean": tensor_proto.BoolVal.AddRange(proto_values.Data()); break; case "Int32": tensor_proto.IntVal.AddRange(proto_values.Data()); break; + case "Int64": + tensor_proto.Int64Val.AddRange(proto_values.Data()); + break; case "Single": tensor_proto.FloatVal.AddRange(proto_values.Data()); break; @@ -286,7 +303,7 @@ namespace Tensorflow default: throw new NotImplementedException("as_shape Not Implemented"); } - dim.Name = $"dim_{i}"; + // dim.Name = $"dim_{i}"; shape.Dim.Add(dim); } @@ -317,7 +334,7 @@ namespace Tensorflow { var dim = new TensorShapeProto.Types.Dim(); dim.Size = tshape.Dimensions[i]; - dim.Name = $"dim_{i}"; + //dim.Name = $"dim_{i}"; shape.Dim.Add(dim); } diff --git a/src/TensorFlowNET.Core/Train/AdamOptimizer.cs b/src/TensorFlowNET.Core/Train/AdamOptimizer.cs new file mode 100644 index 00000000..b6063234 --- /dev/null +++ b/src/TensorFlowNET.Core/Train/AdamOptimizer.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Train +{ + /// + /// Optimizer that implements the Adam algorithm. + /// http://arxiv.org/abs/1412.6980 + /// + public class AdamOptimizer : Optimizer + { + private float _beta1; + private float _beta2; + private float _epsilon; + + public AdamOptimizer(float learning_rate, float beta1 = 0.9f, float beta2 = 0.999f, float epsilon = 1e-8f, bool use_locking = false, string name = "Adam") + : base(learning_rate, use_locking, name) + { + _beta1 = beta1; + _beta2 = beta2; + _epsilon = epsilon; + } + } +} diff --git a/src/TensorFlowNET.Core/Train/GradientDescentOptimizer.cs b/src/TensorFlowNET.Core/Train/GradientDescentOptimizer.cs index 8473d819..ecdb22f6 100644 --- a/src/TensorFlowNET.Core/Train/GradientDescentOptimizer.cs +++ b/src/TensorFlowNET.Core/Train/GradientDescentOptimizer.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow +namespace Tensorflow.Train { public class GradientDescentOptimizer : Optimizer { diff --git a/src/TensorFlowNET.Core/Train/Optimizer.cs b/src/TensorFlowNET.Core/Train/Optimizer.cs index fc76f7b5..9a88601b 100644 --- a/src/TensorFlowNET.Core/Train/Optimizer.cs +++ b/src/TensorFlowNET.Core/Train/Optimizer.cs @@ -34,6 +34,7 @@ namespace Tensorflow Name = name; _use_locking = use_locking; + LearningRate = learning_rate; // Dictionary of slots. _slots = new Dictionary(); _non_slot_dict = new Dictionary(); @@ -49,6 +50,7 @@ namespace Tensorflow /// was not `None`, that operation also increments `global_step`. /// public Operation minimize(Tensor loss, + RefVariable global_step = null, GateGradientType gate_gradients = GateGradientType.GATE_OP, bool colocate_gradients_with_ops = false) { diff --git a/src/TensorFlowNET.Core/Train/Saving/Saver.cs b/src/TensorFlowNET.Core/Train/Saving/Saver.cs index d81908e7..c223ca97 100644 --- a/src/TensorFlowNET.Core/Train/Saving/Saver.cs +++ b/src/TensorFlowNET.Core/Train/Saving/Saver.cs @@ -251,7 +251,7 @@ namespace Tensorflow { return export_meta_graph( filename: filename, - graph_def: ops.get_default_graph()._as_graph_def(add_shapes: true), + graph_def: ops.get_default_graph().as_graph_def(add_shapes: true), saver_def: _saver_def, collection_list: collection_list, as_text: as_text, diff --git a/src/TensorFlowNET.Core/Train/tf.optimizers.cs b/src/TensorFlowNET.Core/Train/tf.optimizers.cs index 5c41dacd..b4925f3a 100644 --- a/src/TensorFlowNET.Core/Train/tf.optimizers.cs +++ b/src/TensorFlowNET.Core/Train/tf.optimizers.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Text; +using Tensorflow.Train; namespace Tensorflow { @@ -11,9 +12,12 @@ namespace Tensorflow { public static Optimizer GradientDescentOptimizer(float learning_rate) => new GradientDescentOptimizer(learning_rate); + public static Optimizer AdamOptimizer(float learning_rate) => new AdamOptimizer(learning_rate); + public static Saver Saver() => new Saver(); - public static string write_graph(Graph graph, string logdir, string name, bool as_text = true) => graph_io.write_graph(graph, logdir, name, as_text); + public static string write_graph(Graph graph, string logdir, string name, bool as_text = true) + => graph_io.write_graph(graph, logdir, name, as_text); public static Saver import_meta_graph(string meta_graph_or_file, bool clear_devices = false, diff --git a/src/TensorFlowNET.Core/Variables/RefVariable.Operators.cs b/src/TensorFlowNET.Core/Variables/RefVariable.Operators.cs index c0b23575..0d27227a 100644 --- a/src/TensorFlowNET.Core/Variables/RefVariable.Operators.cs +++ b/src/TensorFlowNET.Core/Variables/RefVariable.Operators.cs @@ -13,7 +13,8 @@ namespace Tensorflow public static Tensor operator -(RefVariable x, int y) => op_helper("sub", x, y); public static Tensor operator -(RefVariable x, float y) => op_helper("sub", x, y); public static Tensor operator -(RefVariable x, double y) => op_helper("sub", x, y); - + public static Tensor operator -(RefVariable x, Tensor y) => op_helper("sub", x, y); + private static Tensor op_helper(string default_name, RefVariable x, T y) { var tensor1 = x.value(); diff --git a/src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs b/src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs index 737d95b1..8e75c857 100644 --- a/src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs +++ b/src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs @@ -52,7 +52,7 @@ namespace Tensorflow bool use_locking = true, string name = null) { - var _op = _op_def_lib._apply_op_helper("Assign", name: name, args: new { _ref_ = tensor, value, validate_shape, use_locking }); + var _op = _op_def_lib._apply_op_helper("Assign", name: name, args: new { @ref = tensor, value, validate_shape, use_locking }); var _result = _op.outputs; var _inputs_flat = _op.inputs; @@ -66,5 +66,15 @@ namespace Tensorflow return _result[0]; } + + public static Tensor assign_sub(RefVariable @ref, + Tensor value, + bool use_locking = false, + string name = null) + { + var _op = _op_def_lib._apply_op_helper("AssignSub", name: name, args: new { @ref, value, use_locking }); + + return _op.outputs[0]; + } } } diff --git a/src/TensorFlowNET.Core/Variables/state_ops.cs b/src/TensorFlowNET.Core/Variables/state_ops.cs index 0144a138..b2cb6082 100644 --- a/src/TensorFlowNET.Core/Variables/state_ops.cs +++ b/src/TensorFlowNET.Core/Variables/state_ops.cs @@ -24,5 +24,13 @@ namespace Tensorflow name: name, container: container, shared_name: shared_name); + + public static Tensor assign_sub(RefVariable @ref, + Tensor value, + bool use_locking = false, + string name = null) => gen_state_ops.assign_sub(@ref, + value, + use_locking: use_locking, + name: name); } } diff --git a/src/TensorFlowNET.Core/Variables/variable_scope.py.cs b/src/TensorFlowNET.Core/Variables/variable_scope.py.cs index 1a15a6ac..beb5e703 100644 --- a/src/TensorFlowNET.Core/Variables/variable_scope.py.cs +++ b/src/TensorFlowNET.Core/Variables/variable_scope.py.cs @@ -156,6 +156,7 @@ namespace Tensorflow { return new RefVariable(initial_value, trainable: trainable.Value, + validate_shape: validate_shape, name: name, dtype: dtype); } diff --git a/src/TensorFlowNET.Core/Variables/variables.py.cs b/src/TensorFlowNET.Core/Variables/variables.py.cs index 5cde1359..4f11a7a8 100644 --- a/src/TensorFlowNET.Core/Variables/variables.py.cs +++ b/src/TensorFlowNET.Core/Variables/variables.py.cs @@ -47,11 +47,11 @@ namespace Tensorflow /// special tokens filters by prefix. /// /// A list of `Variable` objects. - public static List global_variables(string scope = "") + public static List global_variables(string scope = null) { var result = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope); - return result as List; + return result == null ? new List() : result as List; } /// @@ -62,7 +62,10 @@ namespace Tensorflow /// An Op that run the initializers of all the specified variables. public static Operation variables_initializer(RefVariable[] var_list, string name = "init") { - return control_flow_ops.group(var_list.Select(x => x.initializer).ToArray(), name); + if (var_list.Length > 0) + return control_flow_ops.group(var_list.Select(x => x.initializer).ToArray(), name); + else + return gen_control_flow_ops.no_op(name: name); } } } diff --git a/src/TensorFlowNET.Core/ops.GraphKeys.cs b/src/TensorFlowNET.Core/ops.GraphKeys.cs index 61d527f6..9e1e72f2 100644 --- a/src/TensorFlowNET.Core/ops.GraphKeys.cs +++ b/src/TensorFlowNET.Core/ops.GraphKeys.cs @@ -47,6 +47,10 @@ namespace Tensorflow // Used to store v2 summary names. public static string _SUMMARY_COLLECTION = "_SUMMARY_V2"; + + // Key for control flow context. + public static string COND_CONTEXT = "cond_context"; + public static string WHILE_CONTEXT = "while_context"; } } } diff --git a/src/TensorFlowNET.Core/ops.py.cs b/src/TensorFlowNET.Core/ops.py.cs index 00f1add5..602e0137 100644 --- a/src/TensorFlowNET.Core/ops.py.cs +++ b/src/TensorFlowNET.Core/ops.py.cs @@ -9,6 +9,7 @@ using Google.Protobuf; using System.Linq; using NumSharp.Core; using System.ComponentModel; +using Tensorflow.Gradients; namespace Tensorflow { @@ -40,7 +41,7 @@ namespace Tensorflow /// list contains the values in the order under which they were /// collected. /// - public static object get_collection(string key, string scope = "") + public static object get_collection(string key, string scope = null) { return get_default_graph().get_collection(key, scope); } @@ -345,43 +346,6 @@ namespace Tensorflow session.run(operation, feed_dict); } - public static Func get_gradient_function(Operation op) - { - if (op.inputs == null) return null; - - return (oper, out_grads) => - { - // Console.WriteLine($"get_gradient_function: {oper.type} '{oper.name}'"); - - switch (oper.type) - { - case "Add": - var add = math_grad._AddGrad(oper, out_grads); - return new Tensor[] { add.Item1, add.Item2 }; - case "Identity": - var id = math_grad._IdGrad(oper, out_grads); - return new Tensor[] { id }; - case "Mul": - var mul = math_grad._MulGrad(oper, out_grads); - return new Tensor[] { mul.Item1, mul.Item2 }; - case "Sum": - var sum = math_grad._SumGrad(oper, out_grads); - return new Tensor[] { sum.Item1, sum.Item2 }; - case "Sub": - var sub = math_grad._SubGrad(oper, out_grads); - return new Tensor[] { sub.Item1, sub.Item2 }; - case "Pow": - var pow = math_grad._PowGrad(oper, out_grads); - return new Tensor[] { pow.Item1, pow.Item2 }; - case "RealDiv": - var realdiv = math_grad._RealDivGrad(oper, out_grads); - return new Tensor[] { realdiv.Item1, realdiv.Item2 }; - default: - throw new NotImplementedException($"get_gradient_function {oper.type}"); - } - }; - } - public static Tensor[] convert_n_to_tensor_or_indexed_slices(Tensor[] values, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) { return internal_convert_n_to_tensor_or_indexed_slices(values, dtype: dtype, name: name); @@ -417,13 +381,13 @@ namespace Tensorflow return ret.ToArray(); } - public static Tensor[] internal_convert_n_to_tensor(T[] values, TF_DataType dtype = TF_DataType.DtInvalid, + public static Tensor[] internal_convert_n_to_tensor(object values, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, TF_DataType preferred_dtype = TF_DataType.DtInvalid, bool as_ref = false) { var ret = new List(); - foreach((int i, T value) in Python.enumerate(values)) + foreach((int i, object value) in enumerate(values as object[])) { string n = string.IsNullOrEmpty(name) ? "" : $"{name}_{i}"; ret.Add(internal_convert_to_tensor(value, dtype: dtype, name: n, as_ref: as_ref, preferred_dtype: preferred_dtype)); @@ -434,7 +398,8 @@ namespace Tensorflow public static Tensor internal_convert_to_tensor(object value, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, TF_DataType preferred_dtype = TF_DataType.DtInvalid, - bool as_ref = false) + bool as_ref = false, + string scope = null) { if (dtype == TF_DataType.DtInvalid) dtype = preferred_dtype; @@ -445,24 +410,14 @@ namespace Tensorflow return constant_op.constant(nd, dtype: dtype, name: name); case Tensor tensor: return tensor; - case string str: - return constant_op.constant(str, dtype: dtype, name: name); - case string[] strArray: - return constant_op.constant(strArray, dtype: dtype, name: name); - case int intVal: - return constant_op.constant(intVal, dtype: dtype, name: name); - case int[] intArray: - return constant_op.constant(intArray, dtype: dtype, name: name); - case float floatVal: - return constant_op.constant(floatVal, dtype: dtype, name: name); - case float[] floatArray: - return constant_op.constant(floatArray, dtype: dtype, name: name); - case double doubleVal: - return constant_op.constant(doubleVal, dtype: dtype, name: name); + case Tensor[] tensors: + return array_ops._autopacking_helper(tensors, dtype, name); case RefVariable varVal: return varVal._TensorConversionFunction(as_ref: as_ref); + case object[] objects: + return array_ops._autopacking_conversion_function(objects, dtype: dtype, name: name); default: - throw new NotImplementedException($"internal_convert_to_tensor: Can't convert {value.GetType().Name} to Tensor"); + return constant_op.constant(value, dtype: dtype, name: name); } } diff --git a/src/TensorFlowNET.Core/tf.cs b/src/TensorFlowNET.Core/tf.cs index e1e1331d..65551e7b 100644 --- a/src/TensorFlowNET.Core/tf.cs +++ b/src/TensorFlowNET.Core/tf.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Eager; +using static Tensorflow.ops; namespace Tensorflow { @@ -21,11 +22,13 @@ namespace Tensorflow public static RefVariable Variable(T data, bool trainable = true, + bool validate_shape = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid) { return Tensorflow.variable_scope.default_variable_creator(data, trainable: trainable, + validate_shape: validate_shape, name: name, dtype: TF_DataType.DtInvalid); } diff --git a/tensorflowlib/README.md b/tensorflowlib/README.md index 5dea59ee..2a72b8a5 100644 --- a/tensorflowlib/README.md +++ b/tensorflowlib/README.md @@ -1,3 +1,5 @@ +TensorFlow.NET pack all required libraries in architecture-specific assemblies folders per NuGet standard. + Here are some pre-built TensorFlow binaries you can use for each platform: - Linux @@ -6,7 +8,18 @@ Here are some pre-built TensorFlow binaries you can use for each platform: - Mac: https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.13.1.tar.gz - Windows: https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-1.13.1.zip +### Run in Linux + +`Install-Package TensorFlow.NET` + +Download Linux pre-built library and unzip `libtensorflow.so` and `libtensorflow_framework.so` into current running directory. + +### Run in Mac OS + +### Build from source for Windows + https://www.tensorflow.org/install/source_windows + pacman -S git patch unzip 1. Build static library diff --git a/tensorflowlib/runtimes/linux-x64/native/libtensorflow.so b/tensorflowlib/runtimes/linux-x64/native/libtensorflow.so new file mode 100644 index 00000000..e69de29b diff --git a/tensorflowlib/runtimes/linux-x64/native/libtensorflow_framework.so b/tensorflowlib/runtimes/linux-x64/native/libtensorflow_framework.so new file mode 100644 index 00000000..e69de29b diff --git a/tensorflowlib/runtimes/win-x64/native/tensorflow.dll b/tensorflowlib/runtimes/win-x64/native/tensorflow.dll new file mode 100644 index 00000000..82e86b4e Binary files /dev/null and b/tensorflowlib/runtimes/win-x64/native/tensorflow.dll differ diff --git a/test/TensorFlowNET.Examples/BasicEagerApi.cs b/test/TensorFlowNET.Examples/BasicEagerApi.cs index 2e164bc4..0c4f4bbf 100644 --- a/test/TensorFlowNET.Examples/BasicEagerApi.cs +++ b/test/TensorFlowNET.Examples/BasicEagerApi.cs @@ -11,8 +11,13 @@ namespace TensorFlowNET.Examples /// public class BasicEagerApi : IExample { + public int Priority => 100; + public bool Enabled => false; + public string Name => "Basic Eager"; + private Tensor a, b, c, d; - public void Run() + + public bool Run() { // Set Eager API Console.WriteLine("Setting Eager mode..."); @@ -33,6 +38,12 @@ namespace TensorFlowNET.Examples Console.WriteLine($"a * b = {d}"); // Full compatibility with Numpy + + return true; + } + + public void PrepareData() + { } } } diff --git a/test/TensorFlowNET.Examples/BasicOperations.cs b/test/TensorFlowNET.Examples/BasicOperations.cs index c18904e0..d9025cfe 100644 --- a/test/TensorFlowNET.Examples/BasicOperations.cs +++ b/test/TensorFlowNET.Examples/BasicOperations.cs @@ -10,11 +10,15 @@ namespace TensorFlowNET.Examples /// Basic Operations example using TensorFlow library. /// https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/1_Introduction/basic_operations.py /// - public class BasicOperations : IExample + public class BasicOperations : Python, IExample { + public bool Enabled => true; + public int Priority => 2; + public string Name => "Basic Operations"; + private Session sess; - public void Run() + public bool Run() { // Basic constant operations // The value returned by the constructor represents the output @@ -86,15 +90,16 @@ namespace TensorFlowNET.Examples // graph: the two constants and matmul. // // The output of the op is returned in 'result' as a numpy `ndarray` object. - using (sess = tf.Session()) + return with(tf.Session(), sess => { var result = sess.run(product); Console.WriteLine(result.ToString()); // ==> [[ 12.]] - if (result.Data()[0] != 12) - { - throw new ValueError("BasicOperations"); - } - } + return result.Data()[0] == 12; + }); + } + + public void PrepareData() + { } } } diff --git a/test/TensorFlowNET.Examples/HelloWorld.cs b/test/TensorFlowNET.Examples/HelloWorld.cs index f726bc06..b0ddeb34 100644 --- a/test/TensorFlowNET.Examples/HelloWorld.cs +++ b/test/TensorFlowNET.Examples/HelloWorld.cs @@ -9,9 +9,13 @@ namespace TensorFlowNET.Examples /// Simple hello world using TensorFlow /// https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/1_Introduction/helloworld.py /// - public class HelloWorld : IExample + public class HelloWorld : Python, IExample { - public void Run() + public int Priority => 1; + public bool Enabled => true; + public string Name => "Hello World"; + + public bool Run() { /* Create a Constant op The op is added as a node to the default graph. @@ -22,16 +26,17 @@ namespace TensorFlowNET.Examples var hello = tf.constant(str); // Start tf session - using (var sess = tf.Session()) + return with(tf.Session(), sess => { // Run the op var result = sess.run(hello); Console.WriteLine(result.ToString()); - if(!result.ToString().Equals(str)) - { - throw new ValueError("HelloWorld example acts in unexpected way."); - } - } + return result.ToString().Equals(str); + }); + } + + public void PrepareData() + { } } } diff --git a/test/TensorFlowNET.Examples/IExample.cs b/test/TensorFlowNET.Examples/IExample.cs index 31dd3669..c8320810 100644 --- a/test/TensorFlowNET.Examples/IExample.cs +++ b/test/TensorFlowNET.Examples/IExample.cs @@ -10,6 +10,25 @@ namespace TensorFlowNET.Examples /// public interface IExample { - void Run(); + /// + /// running order + /// + int Priority { get; } + /// + /// True to run example + /// + bool Enabled { get; } + + string Name { get; } + + /// + /// Build dataflow graph, train and predict + /// + /// + bool Run(); + /// + /// Prepare dataset + /// + void PrepareData(); } } diff --git a/test/TensorFlowNET.Examples/ImageRecognition.cs b/test/TensorFlowNET.Examples/ImageRecognition.cs index 47d0ac07..9a5805eb 100644 --- a/test/TensorFlowNET.Examples/ImageRecognition.cs +++ b/test/TensorFlowNET.Examples/ImageRecognition.cs @@ -12,12 +12,16 @@ namespace TensorFlowNET.Examples { public class ImageRecognition : Python, IExample { + public int Priority => 6; + public bool Enabled => true; + public string Name => "Image Recognition"; + string dir = "ImageRecognition"; string pbFile = "tensorflow_inception_graph.pb"; string labelFile = "imagenet_comp_graph_label_strings.txt"; string picFile = "grace_hopper.jpg"; - public void Run() + public bool Run() { PrepareData(); @@ -54,7 +58,10 @@ namespace TensorFlowNET.Examples }); Console.WriteLine($"{picFile}: {labels[idx]} {propability}"); + return labels[idx].Equals("military uniform"); } + + return false; } private NDArray ReadTensorFromImageFile(string file_name, @@ -78,7 +85,7 @@ namespace TensorFlowNET.Examples }); } - private void PrepareData() + public void PrepareData() { Directory.CreateDirectory(dir); diff --git a/test/TensorFlowNET.Examples/InceptionArchGoogLeNet.cs b/test/TensorFlowNET.Examples/InceptionArchGoogLeNet.cs index 75be7738..99581d4f 100644 --- a/test/TensorFlowNET.Examples/InceptionArchGoogLeNet.cs +++ b/test/TensorFlowNET.Examples/InceptionArchGoogLeNet.cs @@ -19,6 +19,10 @@ namespace TensorFlowNET.Examples /// public class InceptionArchGoogLeNet : Python, IExample { + public bool Enabled => false; + public int Priority => 100; + public string Name => "Inception Arch GoogLeNet"; + string dir = "label_image_data"; string pbFile = "inception_v3_2016_08_28_frozen.pb"; string labelFile = "imagenet_slim_labels.txt"; @@ -30,7 +34,7 @@ namespace TensorFlowNET.Examples string input_name = "import/input"; string output_name = "import/InceptionV3/Predictions/Reshape_1"; - public void Run() + public bool Run() { PrepareData(); @@ -60,6 +64,8 @@ namespace TensorFlowNET.Examples foreach (float idx in top_k) Console.WriteLine($"{picFile}: {idx} {labels[(int)idx]}, {results[(int)idx]}"); + + return true; } private NDArray ReadTensorFromImageFile(string file_name, @@ -83,7 +89,7 @@ namespace TensorFlowNET.Examples }); } - private void PrepareData() + public void PrepareData() { Directory.CreateDirectory(dir); diff --git a/test/TensorFlowNET.Examples/KMeansClustering.cs b/test/TensorFlowNET.Examples/KMeansClustering.cs new file mode 100644 index 00000000..1305fef5 --- /dev/null +++ b/test/TensorFlowNET.Examples/KMeansClustering.cs @@ -0,0 +1,52 @@ +using NumSharp.Core; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow; +using Tensorflow.Clustering; +using TensorFlowNET.Examples.Utility; + +namespace TensorFlowNET.Examples +{ + /// + /// Implement K-Means algorithm with TensorFlow.NET, and apply it to classify + /// handwritten digit images. + /// https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/kmeans.py + /// + public class KMeansClustering : Python, IExample + { + public int Priority => 7; + public bool Enabled => true; + public string Name => "K-means Clustering"; + + Datasets mnist; + NDArray full_data_x; + int num_steps = 50; // Total steps to train + int batch_size = 1024; // The number of samples per batch + int k = 25; // The number of clusters + int num_classes = 10; // The 10 digits + int num_features = 784; // Each image is 28x28 pixels + + public bool Run() + { + // Input images + var X = tf.placeholder(tf.float32, shape: new TensorShape(-1, num_features)); + // Labels (for assigning a label to a centroid and testing) + var Y = tf.placeholder(tf.float32, shape: new TensorShape(-1, num_classes)); + + // K-Means Parameters + var kmeans = new KMeans(X, k, distance_metric: KMeans.COSINE_DISTANCE, use_mini_batch: true); + + // Build KMeans graph + var training_graph = kmeans.training_graph(); + + return false; + } + + public void PrepareData() + { + mnist = MnistDataSet.read_data_sets("mnist", one_hot: true); + full_data_x = mnist.train.images; + } + } +} diff --git a/test/TensorFlowNET.Examples/LinearRegression.cs b/test/TensorFlowNET.Examples/LinearRegression.cs index 35d64e5c..7f550c68 100644 --- a/test/TensorFlowNET.Examples/LinearRegression.cs +++ b/test/TensorFlowNET.Examples/LinearRegression.cs @@ -12,6 +12,10 @@ namespace TensorFlowNET.Examples /// public class LinearRegression : Python, IExample { + public int Priority => 3; + public bool Enabled => true; + public string Name => "Linear Regression"; + NumPyRandom rng = np.random; // Parameters @@ -19,14 +23,13 @@ namespace TensorFlowNET.Examples int training_epochs = 1000; int display_step = 50; - public void Run() + NDArray train_X, train_Y; + int n_samples; + + public bool Run() { // Training Data - var train_X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f, - 7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f); - var train_Y = np.array(1.7f, 2.76f, 2.09f, 3.19f, 1.694f, 1.573f, 3.366f, 2.596f, 2.53f, 1.221f, - 2.827f, 3.465f, 1.65f, 2.904f, 2.42f, 2.94f, 1.3f); - var n_samples = train_X.shape[0]; + PrepareData(); // tf Graph Input var X = tf.placeholder(tf.float32); @@ -53,7 +56,7 @@ namespace TensorFlowNET.Examples var init = tf.global_variables_initializer(); // Start training - with(tf.Session(), sess => + return with(tf.Session(), sess => { // Run the initializer sess.run(init); @@ -92,8 +95,20 @@ namespace TensorFlowNET.Examples new FeedItem(X, test_X), new FeedItem(Y, test_Y)); Console.WriteLine($"Testing cost={testing_cost}"); - Console.WriteLine($"Absolute mean square loss difference: {Math.Abs((float)training_cost - (float)testing_cost)}"); + var diff = Math.Abs((float)training_cost - (float)testing_cost); + Console.WriteLine($"Absolute mean square loss difference: {diff}"); + + return diff < 0.01; }); } + + public void PrepareData() + { + train_X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f, + 7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f); + train_Y = np.array(1.7f, 2.76f, 2.09f, 3.19f, 1.694f, 1.573f, 3.366f, 2.596f, 2.53f, 1.221f, + 2.827f, 3.465f, 1.65f, 2.904f, 2.42f, 2.94f, 1.3f); + n_samples = train_X.shape[0]; + } } } diff --git a/test/TensorFlowNET.Examples/LogisticRegression.cs b/test/TensorFlowNET.Examples/LogisticRegression.cs new file mode 100644 index 00000000..44895932 --- /dev/null +++ b/test/TensorFlowNET.Examples/LogisticRegression.cs @@ -0,0 +1,148 @@ +using Newtonsoft.Json; +using NumSharp.Core; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Tensorflow; +using TensorFlowNET.Examples.Utility; + +namespace TensorFlowNET.Examples +{ + /// + /// A logistic regression learning algorithm example using TensorFlow library. + /// This example is using the MNIST database of handwritten digits + /// https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/logistic_regression.py + /// + public class LogisticRegression : Python, IExample + { + public int Priority => 4; + public bool Enabled => true; + public string Name => "Logistic Regression"; + + private float learning_rate = 0.01f; + private int training_epochs = 10; + private int batch_size = 100; + private int display_step = 1; + + Datasets mnist; + + public bool Run() + { + PrepareData(); + + // tf Graph Input + var x = tf.placeholder(tf.float32, new TensorShape(-1, 784)); // mnist data image of shape 28*28=784 + var y = tf.placeholder(tf.float32, new TensorShape(-1, 10)); // 0-9 digits recognition => 10 classes + + // Set model weights + var W = tf.Variable(tf.zeros(new Shape(784, 10))); + var b = tf.Variable(tf.zeros(new Shape(10))); + + // Construct model + var pred = tf.nn.softmax(tf.matmul(x, W) + b); // Softmax + + // Minimize error using cross entropy + var cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(pred), reduction_indices: 1)); + + // Gradient Descent + var optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost); + + // Initialize the variables (i.e. assign their default value) + var init = tf.global_variables_initializer(); + + return with(tf.Session(), sess => + { + + // Run the initializer + sess.run(init); + + // Training cycle + foreach (var epoch in range(training_epochs)) + { + var avg_cost = 0.0f; + var total_batch = mnist.train.num_examples / batch_size; + // Loop over all batches + foreach (var i in range(total_batch)) + { + var (batch_xs, batch_ys) = mnist.train.next_batch(batch_size); + // Run optimization op (backprop) and cost op (to get loss value) + var result = sess.run(new object[] { optimizer, cost }, + new FeedItem(x, batch_xs), + new FeedItem(y, batch_ys)); + + var c = (float)result[1]; + // Compute average loss + avg_cost += c / total_batch; + } + + // Display logs per epoch step + if ((epoch + 1) % display_step == 0) + print($"Epoch: {(epoch + 1).ToString("D4")} cost= {avg_cost.ToString("G9")}"); + } + + print("Optimization Finished!"); + // SaveModel(sess); + + // Test model + var correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)); + // Calculate accuracy + var accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)); + float acc = accuracy.eval(new FeedItem(x, mnist.test.images), new FeedItem(y, mnist.test.labels)); + print($"Accuracy: {acc.ToString("F4")}"); + + return acc > 0.9; + }); + } + + public void PrepareData() + { + mnist = MnistDataSet.read_data_sets("mnist", one_hot: true); + } + + public void SaveModel(Session sess) + { + var saver = tf.train.Saver(); + var save_path = saver.save(sess, "logistic_regression/model.ckpt"); + tf.train.write_graph(sess.graph, "logistic_regression", "model.pbtxt", as_text: true); + + FreezeGraph.freeze_graph(input_graph: "logistic_regression/model.pbtxt", + input_saver: "", + input_binary: false, + input_checkpoint: "logistic_regression/model.ckpt", + output_node_names: "Softmax", + restore_op_name: "save/restore_all", + filename_tensor_name: "save/Const:0", + output_graph: "logistic_regression/model.pb", + clear_devices: true, + initializer_nodes: ""); + } + + public void Predict() + { + var graph = new Graph().as_default(); + graph.Import(Path.Join("logistic_regression", "model.pb")); + + with(tf.Session(graph), sess => + { + // restoring the model + // var saver = tf.train.import_meta_graph("logistic_regression/tensorflowModel.ckpt.meta"); + // saver.restore(sess, tf.train.latest_checkpoint('logistic_regression')); + var pred = graph.OperationByName("Softmax"); + var output = pred.outputs[0]; + var x = graph.OperationByName("Placeholder"); + var input = x.outputs[0]; + + // predict + var (batch_xs, batch_ys) = mnist.train.next_batch(10); + var results = sess.run(output, new FeedItem(input, batch_xs[np.arange(1)])); + + if (results.argmax() == (batch_ys[0] as NDArray).argmax()) + print("predicted OK!"); + else + throw new ValueError("predict error, maybe 90% accuracy"); + }); + } + } +} diff --git a/test/TensorFlowNET.Examples/MetaGraph.cs b/test/TensorFlowNET.Examples/MetaGraph.cs index 7ce74ffc..d05386f2 100644 --- a/test/TensorFlowNET.Examples/MetaGraph.cs +++ b/test/TensorFlowNET.Examples/MetaGraph.cs @@ -9,9 +9,14 @@ namespace TensorFlowNET.Examples { public class MetaGraph : Python, IExample { - public void Run() + public int Priority => 100; + public bool Enabled => false; + public string Name => "Meta Graph"; + + public bool Run() { ImportMetaGraph("my-save-dir/"); + return false; } private void ImportMetaGraph(string dir) @@ -27,5 +32,9 @@ namespace TensorFlowNET.Examples logits: logits); }); } + + public void PrepareData() + { + } } } diff --git a/test/TensorFlowNET.Examples/NaiveBayesClassifier.cs b/test/TensorFlowNET.Examples/NaiveBayesClassifier.cs index 5c4874b2..d3cca828 100644 --- a/test/TensorFlowNET.Examples/NaiveBayesClassifier.cs +++ b/test/TensorFlowNET.Examples/NaiveBayesClassifier.cs @@ -11,9 +11,13 @@ namespace TensorFlowNET.Examples /// https://github.com/nicolov/naive_bayes_tensorflow /// public class NaiveBayesClassifier : Python, IExample - { + { + public int Priority => 100; + public bool Enabled => false; + public string Name => "Naive Bayes Classifier"; + public Normal dist { get; set; } - public void Run() + public bool Run() { var X = np.array(new double[][] { new double[] { 5.1, 3.5},new double[] { 4.9, 3.0 },new double[] { 4.7, 3.2 }, new double[] { 4.6, 3.1 },new double[] { 5.0, 3.6 },new double[] { 5.4, 3.9 }, @@ -170,5 +174,10 @@ namespace TensorFlowNET.Examples // exp to get the actual probabilities return tf.exp(log_prob); } + + public void PrepareData() + { + + } } } diff --git a/test/TensorFlowNET.Examples/NamedEntityRecognition.cs b/test/TensorFlowNET.Examples/NamedEntityRecognition.cs index 556b9415..a6c7f2f1 100644 --- a/test/TensorFlowNET.Examples/NamedEntityRecognition.cs +++ b/test/TensorFlowNET.Examples/NamedEntityRecognition.cs @@ -10,7 +10,16 @@ namespace TensorFlowNET.Examples /// public class NamedEntityRecognition : Python, IExample { - public void Run() + public int Priority => 100; + public bool Enabled => false; + public string Name => "NER"; + + public bool Run() + { + throw new NotImplementedException(); + } + + public void PrepareData() { throw new NotImplementedException(); } diff --git a/test/TensorFlowNET.Examples/NearestNeighbor.cs b/test/TensorFlowNET.Examples/NearestNeighbor.cs new file mode 100644 index 00000000..f8899315 --- /dev/null +++ b/test/TensorFlowNET.Examples/NearestNeighbor.cs @@ -0,0 +1,70 @@ +using NumSharp.Core; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow; +using TensorFlowNET.Examples.Utility; + +namespace TensorFlowNET.Examples +{ + /// + /// A nearest neighbor learning algorithm example + /// This example is using the MNIST database of handwritten digits + /// https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/nearest_neighbor.py + /// + public class NearestNeighbor : Python, IExample + { + public int Priority => 5; + public bool Enabled => true; + public string Name => "Nearest Neighbor"; + Datasets mnist; + NDArray Xtr, Ytr, Xte, Yte; + + public bool Run() + { + // tf Graph Input + var xtr = tf.placeholder(tf.float32, new TensorShape(-1, 784)); + var xte = tf.placeholder(tf.float32, new TensorShape(784)); + + // Nearest Neighbor calculation using L1 Distance + // Calculate L1 Distance + var distance = tf.reduce_sum(tf.abs(tf.add(xtr, tf.negative(xte))), reduction_indices: 1); + // Prediction: Get min distance index (Nearest neighbor) + var pred = tf.arg_min(distance, 0); + + float accuracy = 0f; + // Initialize the variables (i.e. assign their default value) + var init = tf.global_variables_initializer(); + with(tf.Session(), sess => + { + // Run the initializer + sess.run(init); + + PrepareData(); + + foreach(int i in range(Xte.shape[0])) + { + // Get nearest neighbor + long nn_index = sess.run(pred, new FeedItem(xtr, Xtr), new FeedItem(xte, Xte[i])); + // Get nearest neighbor class label and compare it to its true label + print($"Test {i} Prediction: {np.argmax(Ytr[nn_index])} True Class: {np.argmax(Yte[i] as NDArray)}"); + // Calculate accuracy + if (np.argmax(Ytr[nn_index]) == np.argmax(Yte[i] as NDArray)) + accuracy += 1f/ Xte.shape[0]; + } + + print($"Accuracy: {accuracy}"); + }); + + return accuracy > 0.9; + } + + public void PrepareData() + { + mnist = MnistDataSet.read_data_sets("mnist", one_hot: true); + // In this example, we limit mnist data + (Xtr, Ytr) = mnist.train.next_batch(5000); // 5000 for training (nn candidates) + (Xte, Yte) = mnist.test.next_batch(200); // 200 for testing + } + } +} diff --git a/test/TensorFlowNET.Examples/Program.cs b/test/TensorFlowNET.Examples/Program.cs index 8ca5b6db..44448caf 100644 --- a/test/TensorFlowNET.Examples/Program.cs +++ b/test/TensorFlowNET.Examples/Program.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; +using System.Drawing; using System.Linq; using System.Reflection; +using Console = Colorful.Console; namespace TensorFlowNET.Examples { @@ -9,27 +12,44 @@ namespace TensorFlowNET.Examples static void Main(string[] args) { var assembly = Assembly.GetEntryAssembly(); - foreach(Type type in assembly.GetTypes().Where(x => x.GetInterfaces().Contains(typeof(IExample)))) + var errors = new List(); + var success = new List(); + var disabled = new List(); + var examples = assembly.GetTypes() + .Where(x => x.GetInterfaces().Contains(typeof(IExample))) + .Select(x => (IExample)Activator.CreateInstance(x)) + .OrderBy(x => x.Priority) + .ToArray(); + + foreach (IExample example in examples) { - if (args.Length > 0 && !args.Contains(type.Name)) + if (args.Length > 0 && !args.Contains(example.Name)) continue; - Console.WriteLine($"{DateTime.UtcNow} Starting {type.Name}"); - - var example = (IExample)Activator.CreateInstance(type); + Console.WriteLine($"{DateTime.UtcNow} Starting {example.Name}", Color.White); try { - example.Run(); + if (example.Enabled) + if (example.Run()) + success.Add($"Example {example.Priority}: {example.Name}"); + else + errors.Add($"Example {example.Priority}: {example.Name}"); + else + disabled.Add($"Example {example.Priority}: {example.Name}"); } catch (Exception ex) { Console.WriteLine(ex); } - Console.WriteLine($"{DateTime.UtcNow} Completed {type.Name}"); + Console.WriteLine($"{DateTime.UtcNow} Completed {example.Name}", Color.White); } + success.ForEach(x => Console.WriteLine($"{x} is OK!", Color.Green)); + disabled.ForEach(x => Console.WriteLine($"{x} is Disabled!", Color.Tan)); + errors.ForEach(x => Console.WriteLine($"{x} is Failed!", Color.Red)); + Console.ReadLine(); } } diff --git a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj index 9e83fdae..59fc1fd0 100644 --- a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj +++ b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj @@ -6,23 +6,14 @@ - + + + - - - - - C:\Program Files\dotnet\sdk\NuGetFallbackFolder\newtonsoft.json\9.0.1\lib\netstandard1.0\Newtonsoft.Json.dll - - - C:\Users\bpeng\Desktop\BoloReborn\NumSharp\src\NumSharp.Core\bin\Debug\netstandard2.0\NumSharp.Core.dll - - - diff --git a/test/TensorFlowNET.Examples/TextClassification/TextClassificationTrain.cs b/test/TensorFlowNET.Examples/TextClassification/TextClassificationTrain.cs index e6e26ba6..a1251750 100644 --- a/test/TensorFlowNET.Examples/TextClassification/TextClassificationTrain.cs +++ b/test/TensorFlowNET.Examples/TextClassification/TextClassificationTrain.cs @@ -15,34 +15,31 @@ namespace TensorFlowNET.Examples.CnnTextClassification /// public class TextClassificationTrain : Python, IExample { + public int Priority => 100; + public bool Enabled => false; + public string Name => "Text Classification"; + private string dataDir = "text_classification"; private string dataFileName = "dbpedia_csv.tar.gz"; private const int CHAR_MAX_LEN = 1014; private const int NUM_CLASS = 2; - public void Run() + public bool Run() { - download_dbpedia(); + PrepareData(); Console.WriteLine("Building dataset..."); var (x, y, alphabet_size) = DataHelpers.build_char_dataset("train", "vdcnn", CHAR_MAX_LEN); var (train_x, valid_x, train_y, valid_y) = train_test_split(x, y, test_size: 0.15f); - with(tf.Session(), sess => + return with(tf.Session(), sess => { new VdCnn(alphabet_size, CHAR_MAX_LEN, NUM_CLASS); - + return false; }); } - public void download_dbpedia() - { - string url = "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz"; - Web.Download(url, dataDir, dataFileName); - Compress.ExtractTGZ(Path.Join(dataDir, dataFileName), dataDir); - } - private (int[][], int[][], int[], int[]) train_test_split(int[][] x, int[] y, float test_size = 0.3f) { int len = x.Length; @@ -75,5 +72,12 @@ namespace TensorFlowNET.Examples.CnnTextClassification return (train_x.ToArray(), valid_x.ToArray(), train_y.ToArray(), valid_y.ToArray()); } + + public void PrepareData() + { + string url = "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz"; + Web.Download(url, dataDir, dataFileName); + Compress.ExtractTGZ(Path.Join(dataDir, dataFileName), dataDir); + } } } diff --git a/test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs b/test/TensorFlowNET.Examples/TextClassification/TextClassificationWithMovieReviews.cs similarity index 88% rename from test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs rename to test/TensorFlowNET.Examples/TextClassification/TextClassificationWithMovieReviews.cs index cd59287b..a1e0fd74 100644 --- a/test/TensorFlowNET.Examples/TextClassificationWithMovieReviews.cs +++ b/test/TensorFlowNET.Examples/TextClassification/TextClassificationWithMovieReviews.cs @@ -11,12 +11,17 @@ namespace TensorFlowNET.Examples { public class TextClassificationWithMovieReviews : Python, IExample { + public int Priority => 7; + public bool Enabled => false; + public string Name => "Movie Reviews"; + string dir = "text_classification_with_movie_reviews"; string dataFile = "imdb.zip"; + NDArray train_data, train_labels, test_data, test_labels; - public void Run() + public bool Run() { - var((train_data, train_labels), (test_data, test_labels)) = PrepareData(); + PrepareData(); Console.WriteLine($"Training entries: {train_data.size}, labels: {train_labels.size}"); @@ -37,9 +42,12 @@ namespace TensorFlowNET.Examples int vocab_size = 10000; var model = keras.Sequential(); + model.add(keras.layers.Embedding(vocab_size, 16)); + + return false; } - private ((NDArray, NDArray), (NDArray, NDArray)) PrepareData() + public void PrepareData() { Directory.CreateDirectory(dir); @@ -70,7 +78,11 @@ namespace TensorFlowNET.Examples var y_train = labels_train; var y_test = labels_test; - return ((x_train, y_train), (x_test, y_test)); + x_train = train_data; + train_labels = y_train; + + test_data = x_test; + test_labels = y_test; } private NDArray ReadData(string file) diff --git a/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs b/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs index 05929d3d..d01b458d 100644 --- a/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs +++ b/test/TensorFlowNET.Examples/TextClassification/cnn_models/VdCnn.cs @@ -14,6 +14,7 @@ namespace TensorFlowNET.Examples.TextClassification private int[] num_blocks; private float learning_rate; private IInitializer cnn_initializer; + private IInitializer fc_initializer; private Tensor x; private Tensor y; private Tensor is_training; @@ -21,6 +22,9 @@ namespace TensorFlowNET.Examples.TextClassification private RefVariable embeddings; private Tensor x_emb; private Tensor x_expanded; + private Tensor logits; + private Tensor predictions; + private Tensor loss; public VdCnn(int alphabet_size, int document_max_len, int num_class) { @@ -30,6 +34,8 @@ namespace TensorFlowNET.Examples.TextClassification num_blocks = new int[] { 2, 2, 2, 2 }; learning_rate = 0.001f; cnn_initializer = tf.keras.initializers.he_normal(); + fc_initializer = tf.truncated_normal_initializer(stddev: 0.05f); + x = tf.placeholder(tf.int32, new TensorShape(-1, document_max_len), name: "x"); y = tf.placeholder(tf.int32, new TensorShape(-1), name: "y"); is_training = tf.placeholder(tf.boolean, new TensorShape(), name: "is_training"); @@ -46,6 +52,12 @@ namespace TensorFlowNET.Examples.TextClassification Tensor conv0 = null; Tensor conv1 = null; + Tensor conv2 = null; + Tensor conv3 = null; + Tensor conv4 = null; + Tensor h_flat = null; + Tensor fc1_out = null; + Tensor fc2_out = null; // First Convolution Layer with(tf.variable_scope("conv-0"), delegate @@ -62,7 +74,58 @@ namespace TensorFlowNET.Examples.TextClassification with(tf.name_scope("conv-block-1"), delegate { conv1 = conv_block(conv0, 1); }); - + + with(tf.name_scope("conv-block-2"), delegate { + conv2 = conv_block(conv1, 2); + }); + + with(tf.name_scope("conv-block-3"), delegate { + conv3 = conv_block(conv2, 3); + }); + + with(tf.name_scope("conv-block-4"), delegate + { + conv4 = conv_block(conv3, 4, max_pool: false); + }); + + // ============= k-max Pooling ============= + with(tf.name_scope("k-max-pooling"), delegate + { + var h = tf.transpose(tf.squeeze(conv4, new int[] { -1 }), new int[] { 0, 2, 1 }); + var top_k = tf.nn.top_k(h, k: 8, sorted: false)[0]; + h_flat = tf.reshape(top_k, new int[] { -1, 512 * 8 }); + }); + + // ============= Fully Connected Layers ============= + with(tf.name_scope("fc-1"), scope => + { + fc1_out = tf.layers.dense(h_flat, 2048, activation: tf.nn.relu, kernel_initializer: fc_initializer); + }); + + with(tf.name_scope("fc-2"), scope => + { + fc2_out = tf.layers.dense(fc1_out, 2048, activation: tf.nn.relu, kernel_initializer: fc_initializer); + }); + + with(tf.name_scope("fc-3"), scope => + { + logits = tf.layers.dense(fc2_out, num_class, activation: null, kernel_initializer: fc_initializer); + predictions = tf.argmax(logits, -1, output_type: tf.int32); + }); + + // ============= Loss and Accuracy ============= + with(tf.name_scope("loss"), delegate + { + var y_one_hot = tf.one_hot(y, num_class); + loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits: logits, labels: y_one_hot)); + + var update_ops = tf.get_collection(ops.GraphKeys.UPDATE_OPS) as List; + with(tf.control_dependencies(update_ops.Select(x => (Operation)x).ToArray()), delegate + { + var adam = tf.train.AdamOptimizer(learning_rate); + adam.minimize(loss, global_step: global_step); + }); + }); } private Tensor conv_block(Tensor input, int i, bool max_pool = true) @@ -93,7 +156,11 @@ namespace TensorFlowNET.Examples.TextClassification if (max_pool) { // Max pooling - throw new NotImplementedException("conv_block"); + return tf.layers.max_pooling2d( + conv, + pool_size: new int[] { 3, 1 }, + strides: new int[] { 2, 1 }, + padding: "SAME"); } else { diff --git a/test/TensorFlowNET.Examples/Utility/Compress.cs b/test/TensorFlowNET.Examples/Utility/Compress.cs index cf40e2c4..bc38434b 100644 --- a/test/TensorFlowNET.Examples/Utility/Compress.cs +++ b/test/TensorFlowNET.Examples/Utility/Compress.cs @@ -1,4 +1,5 @@ -using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Core; +using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; using System; using System.IO; @@ -11,6 +12,26 @@ namespace TensorFlowNET.Examples.Utility { public class Compress { + public static void ExtractGZip(string gzipFileName, string targetDir) + { + // Use a 4K buffer. Any larger is a waste. + byte[] dataBuffer = new byte[4096]; + + using (System.IO.Stream fs = new FileStream(gzipFileName, FileMode.Open, FileAccess.Read)) + { + using (GZipInputStream gzipStream = new GZipInputStream(fs)) + { + // Change this to your needs + string fnOut = Path.Combine(targetDir, Path.GetFileNameWithoutExtension(gzipFileName)); + + using (FileStream fsOut = File.Create(fnOut)) + { + StreamUtils.Copy(gzipStream, fsOut, dataBuffer); + } + } + } + } + public static void UnZip(String gzArchiveName, String destFolder) { var flag = gzArchiveName.Split(Path.DirectorySeparatorChar).Last().Split('.').First() + ".bin"; diff --git a/test/TensorFlowNET.Examples/Utility/DataSet.cs b/test/TensorFlowNET.Examples/Utility/DataSet.cs new file mode 100644 index 00000000..59b86a63 --- /dev/null +++ b/test/TensorFlowNET.Examples/Utility/DataSet.cs @@ -0,0 +1,86 @@ +using NumSharp.Core; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow; + +namespace TensorFlowNET.Examples.Utility +{ + public class DataSet + { + private int _num_examples; + public int num_examples => _num_examples; + private int _epochs_completed; + public int epochs_completed => _epochs_completed; + private int _index_in_epoch; + public int index_in_epoch => _index_in_epoch; + private NDArray _images; + public NDArray images => _images; + private NDArray _labels; + public NDArray labels => _labels; + + public DataSet(NDArray images, NDArray labels, TF_DataType dtype, bool reshape) + { + _num_examples = images.shape[0]; + images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]); + images.astype(dtype.as_numpy_datatype()); + images = np.multiply(images, 1.0f / 255.0f); + + labels.astype(dtype.as_numpy_datatype()); + + _images = images; + _labels = labels; + _epochs_completed = 0; + _index_in_epoch = 0; + } + + public (NDArray, NDArray) next_batch(int batch_size, bool fake_data = false, bool shuffle = true) + { + var start = _index_in_epoch; + // Shuffle for the first epoch + if(_epochs_completed == 0 && start == 0 && shuffle) + { + var perm0 = np.arange(_num_examples); + np.random.shuffle(perm0); + _images = images[perm0]; + _labels = labels[perm0]; + } + + // Go to the next epoch + if (start + batch_size > _num_examples) + { + // Finished epoch + _epochs_completed += 1; + + // Get the rest examples in this epoch + var rest_num_examples = _num_examples - start; + var images_rest_part = _images[np.arange(start, _num_examples)]; + var labels_rest_part = _labels[np.arange(start, _num_examples)]; + // Shuffle the data + if (shuffle) + { + var perm = np.arange(_num_examples); + np.random.shuffle(perm); + _images = images[perm]; + _labels = labels[perm]; + } + + start = 0; + _index_in_epoch = batch_size - rest_num_examples; + var end = _index_in_epoch; + var images_new_part = _images[np.arange(start, end)]; + var labels_new_part = _labels[np.arange(start, end)]; + + /*return (np.concatenate(new float[][] { images_rest_part.Data(), images_new_part.Data() }, axis: 0), + np.concatenate(new float[][] { labels_rest_part.Data(), labels_new_part.Data() }, axis: 0));*/ + return (images_new_part, labels_new_part); + } + else + { + _index_in_epoch += batch_size; + var end = _index_in_epoch; + return (_images[np.arange(start, end)], _labels[np.arange(start, end)]); + } + } + } +} diff --git a/test/TensorFlowNET.Examples/Utility/Datasets.cs b/test/TensorFlowNET.Examples/Utility/Datasets.cs new file mode 100644 index 00000000..660e40db --- /dev/null +++ b/test/TensorFlowNET.Examples/Utility/Datasets.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace TensorFlowNET.Examples.Utility +{ + public class Datasets + { + private DataSet _train; + public DataSet train => _train; + + private DataSet _validation; + public DataSet validation => _validation; + + private DataSet _test; + public DataSet test => _test; + + public Datasets(DataSet train, DataSet validation, DataSet test) + { + _train = train; + _validation = validation; + _test = test; + } + } +} diff --git a/test/TensorFlowNET.Examples/Utility/MnistDataSet.cs b/test/TensorFlowNET.Examples/Utility/MnistDataSet.cs new file mode 100644 index 00000000..f54fd95c --- /dev/null +++ b/test/TensorFlowNET.Examples/Utility/MnistDataSet.cs @@ -0,0 +1,114 @@ +using ICSharpCode.SharpZipLib.Core; +using ICSharpCode.SharpZipLib.GZip; +using NumSharp.Core; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Tensorflow; + +namespace TensorFlowNET.Examples.Utility +{ + public class MnistDataSet + { + private const string DEFAULT_SOURCE_URL = "https://storage.googleapis.com/cvdf-datasets/mnist/"; + private const string TRAIN_IMAGES = "train-images-idx3-ubyte.gz"; + private const string TRAIN_LABELS = "train-labels-idx1-ubyte.gz"; + private const string TEST_IMAGES = "t10k-images-idx3-ubyte.gz"; + private const string TEST_LABELS = "t10k-labels-idx1-ubyte.gz"; + + public static Datasets read_data_sets(string train_dir, + bool one_hot = false, + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool reshape = true, + int validation_size = 5000, + string source_url = DEFAULT_SOURCE_URL) + { + Web.Download(source_url + TRAIN_IMAGES, train_dir, TRAIN_IMAGES); + Compress.ExtractGZip(Path.Join(train_dir, TRAIN_IMAGES), train_dir); + var train_images = extract_images(Path.Join(train_dir, TRAIN_IMAGES.Split('.')[0])); + + Web.Download(source_url + TRAIN_LABELS, train_dir, TRAIN_LABELS); + Compress.ExtractGZip(Path.Join(train_dir, TRAIN_LABELS), train_dir); + var train_labels = extract_labels(Path.Join(train_dir, TRAIN_LABELS.Split('.')[0]), one_hot: one_hot); + + Web.Download(source_url + TEST_IMAGES, train_dir, TEST_IMAGES); + Compress.ExtractGZip(Path.Join(train_dir, TEST_IMAGES), train_dir); + var test_images = extract_images(Path.Join(train_dir, TEST_IMAGES.Split('.')[0])); + + Web.Download(source_url + TEST_LABELS, train_dir, TEST_LABELS); + Compress.ExtractGZip(Path.Join(train_dir, TEST_LABELS), train_dir); + var test_labels = extract_labels(Path.Join(train_dir, TEST_LABELS.Split('.')[0]), one_hot: one_hot); + + int end = train_images.shape[0]; + var validation_images = train_images[np.arange(validation_size)]; + var validation_labels = train_labels[np.arange(validation_size)]; + train_images = train_images[np.arange(validation_size, end)]; + train_labels = train_labels[np.arange(validation_size, end)]; + + var train = new DataSet(train_images, train_labels, dtype, reshape); + var validation = new DataSet(validation_images, validation_labels, dtype, reshape); + var test = new DataSet(test_images, test_labels, dtype, reshape); + + return new Datasets(train, validation, test); + } + + public static NDArray extract_images(string file) + { + using (var bytestream = new FileStream(file, FileMode.Open)) + { + var magic = _read32(bytestream); + if (magic != 2051) + throw new ValueError($"Invalid magic number {magic} in MNIST image file: {file}"); + var num_images = _read32(bytestream); + var rows = _read32(bytestream); + var cols = _read32(bytestream); + var buf = new byte[rows * cols * num_images]; + bytestream.Read(buf, 0, buf.Length); + var data = np.frombuffer(buf, np.uint8); + data = data.reshape((int)num_images, (int)rows, (int)cols, 1); + return data; + } + } + + public static NDArray extract_labels(string file, bool one_hot = false, int num_classes = 10) + { + using (var bytestream = new FileStream(file, FileMode.Open)) + { + var magic = _read32(bytestream); + if (magic != 2049) + throw new ValueError($"Invalid magic number {magic} in MNIST label file: {file}"); + var num_items = _read32(bytestream); + var buf = new byte[num_items]; + bytestream.Read(buf, 0, buf.Length); + var labels = np.frombuffer(buf, np.uint8); + if (one_hot) + return dense_to_one_hot(labels, num_classes); + return labels; + } + } + + private static NDArray dense_to_one_hot(NDArray labels_dense, int num_classes) + { + var num_labels = labels_dense.shape[0]; + var index_offset = np.arange(num_labels) * num_classes; + var labels_one_hot = np.zeros(num_labels, num_classes); + + for(int row = 0; row < num_labels; row++) + { + var col = labels_dense.Data(row); + labels_one_hot[row, col] = 1; + } + + return labels_one_hot; + } + + private static uint _read32(FileStream bytestream) + { + var buffer = new byte[sizeof(uint)]; + var count = bytestream.Read(buffer, 0, 4); + return np.frombuffer(buffer, ">u4").Data(0); + } + } +} diff --git a/test/TensorFlowNET.Examples/python/logistic_regression.py b/test/TensorFlowNET.Examples/python/logistic_regression.py new file mode 100644 index 00000000..236d83d1 --- /dev/null +++ b/test/TensorFlowNET.Examples/python/logistic_regression.py @@ -0,0 +1,100 @@ +''' +A logistic regression learning algorithm example using TensorFlow library. +This example is using the MNIST database of handwritten digits +(http://yann.lecun.com/exdb/mnist/) +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf + +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +# Parameters +learning_rate = 0.01 +training_epochs = 10 +batch_size = 100 +display_step = 1 + +# tf Graph Input +x = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784 +y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes + +# Set model weights +W = tf.Variable(tf.zeros([784, 10])) +b = tf.Variable(tf.zeros([10])) + +# Construct model +pred = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax + +# Minimize error using cross entropy +cost = tf.reduce_mean(-tf.reduce_sum(y*tf.log(pred), reduction_indices=1)) +# Gradient Descent +optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) + +# Initialize the variables (i.e. assign their default value) +init = tf.global_variables_initializer() + +# Start training +with tf.Session() as sess: + + # Run the initializer + sess.run(init) + + # Training cycle + for epoch in range(training_epochs): + avg_cost = 0. + total_batch = int(mnist.train.num_examples/batch_size) + # Loop over all batches + for i in range(total_batch): + batch_xs, batch_ys = mnist.train.next_batch(batch_size) + # Run optimization op (backprop) and cost op (to get loss value) + _, c = sess.run([optimizer, cost], feed_dict={x: batch_xs, + y: batch_ys}) + # Compute average loss + avg_cost += c / total_batch + # Display logs per epoch step + if (epoch+1) % display_step == 0: + print("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost)) + + print("Optimization Finished!") + + # Test model + correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) + # Calculate accuracy + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels})) + + # predict + # results = sess.run(pred, feed_dict={x: batch_xs[:1]}) + + # save model + saver = tf.train.Saver() + save_path = saver.save(sess, "logistic_regression/model.ckpt") + tf.train.write_graph(sess.graph.as_graph_def(),'logistic_regression','model.pbtxt', as_text=True) + + freeze_graph.freeze_graph(input_graph = 'logistic_regression/model.pbtxt', + input_saver = "", + input_binary = False, + input_checkpoint = 'logistic_regression/model.ckpt', + output_node_names = "Softmax", + restore_op_name = "save/restore_all", + filename_tensor_name = "save/Const:0", + output_graph = 'logistic_regression/model.pb', + clear_devices = True, + initializer_nodes = "") + + # restoring the model + saver = tf.train.import_meta_graph('logistic_regression/tensorflowModel.ckpt.meta') + saver.restore(sess,tf.train.latest_checkpoint('logistic_regression')) + + # predict + # pred = graph._nodes_by_name["Softmax"] + # output = pred.outputs[0] + # x = graph._nodes_by_name["Placeholder"] + # input = x.outputs[0] + # results = sess.run(output, feed_dict={input: batch_xs[:1]}) \ No newline at end of file diff --git a/test/TensorFlowNET.UnitTest/ConstantTest.cs b/test/TensorFlowNET.UnitTest/ConstantTest.cs index ee496e7c..98329867 100644 --- a/test/TensorFlowNET.UnitTest/ConstantTest.cs +++ b/test/TensorFlowNET.UnitTest/ConstantTest.cs @@ -81,7 +81,7 @@ namespace TensorFlowNET.UnitTest Assert.AreEqual(result.shape[0], 2); Assert.AreEqual(result.shape[1], 3); - Assert.IsTrue(Enumerable.SequenceEqual(new int[] { 3, 2, 1, 1, 1, 3 }, data)); + Assert.IsTrue(Enumerable.SequenceEqual(new int[] { 3, 1, 1, 2, 1, 3 }, data)); }); } diff --git a/test/TensorFlowNET.UnitTest/Eager/CApiVariableTest.cs b/test/TensorFlowNET.UnitTest/Eager/CApiVariableTest.cs index 1a5cb1a5..6e8976b6 100644 --- a/test/TensorFlowNET.UnitTest/Eager/CApiVariableTest.cs +++ b/test/TensorFlowNET.UnitTest/Eager/CApiVariableTest.cs @@ -17,7 +17,7 @@ namespace TensorFlowNET.UnitTest.Eager ContextOptions opts = new ContextOptions(); Context ctx; - [TestMethod] + //[TestMethod] public void Variables() { ctx = new Context(opts, status); diff --git a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj index c9b02eab..af1f38a2 100644 --- a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj @@ -19,12 +19,11 @@ - - + + - diff --git a/test/TensorFlowNET.UnitTest/TensorTest.cs b/test/TensorFlowNET.UnitTest/TensorTest.cs index a1733002..740ed8ad 100644 --- a/test/TensorFlowNET.UnitTest/TensorTest.cs +++ b/test/TensorFlowNET.UnitTest/TensorTest.cs @@ -60,7 +60,7 @@ namespace TensorFlowNET.UnitTest EXPECT_EQ((int)tensor.shape[0], nd.shape[0]); EXPECT_EQ((int)tensor.shape[1], nd.shape[1]); EXPECT_EQ(tensor.bytesize, (ulong)nd.size * sizeof(float)); - Assert.IsTrue(Enumerable.SequenceEqual(nd.Data(), new float[] { 1, 4, 2, 5, 3, 6 })); + Assert.IsTrue(Enumerable.SequenceEqual(nd.Data(), new float[] { 1, 2, 3, 4, 5, 6 })); } ///