From d94c685dba464188f431958073e5d0d1f00ffa27 Mon Sep 17 00:00:00 2001 From: Oceania2018 Date: Mon, 18 Mar 2019 22:28:00 -0500 Subject: [PATCH] Add node to connectivity between two layers. --- src/TensorFlowNET.Core/APIs/keras.layers.cs | 6 +- .../Keras/Engine/Sequential.cs | 6 ++ .../Keras/Layers/Embedding.cs | 13 +++- .../Keras/Layers/InputLayer.cs | 13 +++- src/TensorFlowNET.Core/Keras/Layers/Layer.cs | 32 ++++++--- src/TensorFlowNET.Core/Keras/Layers/Node.cs | 71 +++++++++++++++++++ .../{Engine => Utils}/base_layer_utils.cs | 11 ++- src/TensorFlowNET.Core/Layers/Layer.cs | 2 +- .../TensorFlowNET.Core.csproj | 2 +- .../TensorFlowNET.Examples.csproj | 2 +- .../TensorFlowNET.UnitTest.csproj | 2 +- 11 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Layers/Node.cs rename src/TensorFlowNET.Core/Keras/{Engine => Utils}/base_layer_utils.cs (66%) diff --git a/src/TensorFlowNET.Core/APIs/keras.layers.cs b/src/TensorFlowNET.Core/APIs/keras.layers.cs index 54a1032f..00fe7ee1 100644 --- a/src/TensorFlowNET.Core/APIs/keras.layers.cs +++ b/src/TensorFlowNET.Core/APIs/keras.layers.cs @@ -18,7 +18,7 @@ namespace Tensorflow embeddings_initializer, mask_zero); - public static InputLayer Input(int[] batch_shape = null, + public static Tensor[] Input(int[] batch_shape = null, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool sparse = false, @@ -35,7 +35,9 @@ namespace Tensorflow sparse: sparse, input_tensor: tensor); - throw new NotImplementedException(""); + var outputs = input_layer.inbound_nodes[0].output_tensors; + + return outputs; } } } diff --git a/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs b/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs index 0801cdec..587a956a 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/Sequential.cs @@ -33,6 +33,12 @@ namespace Tensorflow.Keras.Engine batch_shape: batch_shape, dtype: dtype, name: layer._name + "_input"); + + // This will build the current layer + // and create the node connecting the current layer + // to the input layer we just created. + layer.__call__(x); + set_inputs = true; } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs b/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs index 494cad8b..3ea6d65d 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Embedding.cs @@ -9,6 +9,8 @@ namespace Tensorflow.Keras.Layers private int input_dim; private int output_dim; private bool mask_zero; + public RefVariable embeddings; + public IInitializer embeddings_initializer; public Embedding(int input_dim, int output_dim, IInitializer embeddings_initializer = null, @@ -18,10 +20,17 @@ namespace Tensorflow.Keras.Layers { this.input_dim = input_dim; this.output_dim = output_dim; - if (embeddings_initializer == null) - embeddings_initializer = tf.uniform_initializer; + this.embeddings_initializer = embeddings_initializer == null ? tf.uniform_initializer : embeddings_initializer; this.mask_zero = mask_zero; supports_masking = mask_zero; } + + protected override void build(TensorShape input_shape) + { + embeddings = add_weight(shape: new int[] { input_dim, output_dim }, + initializer: embeddings_initializer, + name: "embeddings"); + built = true; + } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs b/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs index e811776d..60257ee0 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs @@ -11,6 +11,7 @@ namespace Tensorflow.Keras.Layers { public bool sparse; public int? batch_size; + public bool is_placeholder; public InputLayer(int[] input_shape = null, int? batch_size = null, @@ -24,7 +25,7 @@ namespace Tensorflow.Keras.Layers this.batch_size = batch_size; this.supports_masking = true; - if(input_tensor == null) + if (input_tensor == null) { var batch_input_shape = new int[] { batch_size.HasValue ? batch_size.Value : -1, -1 }; @@ -39,7 +40,17 @@ namespace Tensorflow.Keras.Layers dtype: dtype, name: name); } + + is_placeholder = true; + _batch_input_shape = batch_input_shape; } + + new Node(this, + inbound_layers: new Layer[0], + node_indices: new int[0], + tensor_indices: new int[0], + input_tensors: new Tensor[] { input_tensor }, + output_tensors: new Tensor[] { input_tensor }); } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/Layer.cs b/src/TensorFlowNET.Core/Keras/Layers/Layer.cs index 3595087f..4b8eebba 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Layer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Layer.cs @@ -39,6 +39,12 @@ namespace Tensorflow.Keras.Layers protected List _updates; public int[] _batch_input_shape; + private List _inbound_nodes; + public List inbound_nodes => _inbound_nodes; + + private List _outbound_nodes; + public List outbound_nodes => _outbound_nodes; + public Layer(bool trainable = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid, @@ -59,13 +65,15 @@ namespace Tensorflow.Keras.Layers _batch_input_shape = new int[] { -1, -1 }; _dtype = dtype; + + _inbound_nodes = new List(); } - public Tensor __call__(Tensor inputs, + public Tensor __call__(Tensor[] inputs, Tensor training = null, VariableScope scope = null) { - var input_list = new Tensor[] { inputs }; + var input_list = inputs; Tensor outputs = null; // We will attempt to build a TF graph if & only if all inputs are symbolic. @@ -88,9 +96,9 @@ namespace Tensorflow.Keras.Layers // Symbolic execution on symbolic tensors. We will attempt to build // the corresponding TF subgraph inside `backend.get_graph()` var graph = backend.get_graph(); - outputs = call(inputs, training: training); - _handle_activity_regularization(inputs, outputs); - _set_mask_metadata(inputs, outputs, null); + outputs = call(inputs[0], training: training); + _handle_activity_regularization(inputs[0], outputs); + _set_mask_metadata(inputs[0], outputs, null); } }); @@ -125,10 +133,10 @@ namespace Tensorflow.Keras.Layers return null; } - protected void _maybe_build(Tensor inputs) + protected void _maybe_build(Tensor[] inputs) { - var input_list = new Tensor[] { inputs }; - build(inputs.getShape()); + var input_list = inputs; + build(input_list[0].getShape()); } protected virtual void build(TensorShape input_shape) @@ -143,10 +151,16 @@ namespace Tensorflow.Keras.Layers bool? trainable = null, Func getter = null) { + if (dtype == TF_DataType.DtInvalid) + dtype = TF_DataType.TF_FLOAT; + + if (trainable == null) + trainable = true; + var variable = _add_variable_with_custom_getter(name, shape, dtype: dtype, - getter: getter, + getter: getter == null ? base_layer_utils.make_variable : getter, overwrite: true, initializer: initializer, trainable: trainable.Value); diff --git a/src/TensorFlowNET.Core/Keras/Layers/Node.cs b/src/TensorFlowNET.Core/Keras/Layers/Node.cs new file mode 100644 index 00000000..d4144c62 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Node.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tensorflow.Keras.Layers +{ + /// + /// A `Node` describes the connectivity between two layers. + /// + public class Node + { + public InputLayer outbound_layer; + public Layer[] inbound_layers; + public int[] node_indices; + public int[] tensor_indices; + public Tensor[] input_tensors; + public Tensor[] output_tensors; + public int[][] input_shapes; + public int[][] output_shapes; + + /// + /// + /// + /// + /// the layer that takes + /// `input_tensors` and turns them into `output_tensors` + /// (the node gets created when the `call` + /// method of the layer was called). + /// + /// + /// a list of layers, the same length as `input_tensors`, + /// the layers from where `input_tensors` originate. + /// + /// + /// a list of integers, the same length as `inbound_layers`. + /// `node_indices[i]` is the origin node of `input_tensors[i]` + /// (necessary since each inbound layer might have several nodes, + /// e.g. if the layer is being shared with a different data stream). + /// + /// + /// list of input tensors. + /// list of output tensors. + public Node(InputLayer outbound_layer, + Layer[] inbound_layers, + int[] node_indices, + int[] tensor_indices, + Tensor[] input_tensors, + Tensor[] output_tensors) + { + this.outbound_layer = outbound_layer; + this.inbound_layers = inbound_layers; + this.node_indices = node_indices; + this.tensor_indices = tensor_indices; + this.input_tensors = input_tensors; + this.output_tensors = output_tensors; + + input_shapes = input_tensors.Select(x => x._shape_tuple()).ToArray(); + output_shapes = output_tensors.Select(x => x._shape_tuple()).ToArray(); + + // Add nodes to all layers involved. + foreach (var layer in inbound_layers) + { + if (layer != null) + layer.outbound_nodes.Add(this); + } + + outbound_layer.inbound_nodes.Add(this); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs b/src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs similarity index 66% rename from src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs rename to src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs index 1f397425..682760f0 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs +++ b/src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs @@ -2,10 +2,19 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.Engine +namespace Tensorflow.Keras.Utils { public class base_layer_utils { + public static RefVariable make_variable(string name, + int[] shape, + TF_DataType dtype = TF_DataType.TF_FLOAT, + IInitializer initializer = null, + bool trainable = false) + { + throw new NotImplementedException(""); + } + /// /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph. /// diff --git a/src/TensorFlowNET.Core/Layers/Layer.cs b/src/TensorFlowNET.Core/Layers/Layer.cs index aa2a7405..4b2d9cf3 100644 --- a/src/TensorFlowNET.Core/Layers/Layer.cs +++ b/src/TensorFlowNET.Core/Layers/Layer.cs @@ -52,7 +52,7 @@ namespace Tensorflow.Layers Python.with(scope_context_manager, scope2 => _current_scope = scope2); // Actually call layer - var outputs = base.__call__(inputs, training: training); + var outputs = base.__call__(new Tensor[] { inputs }, training: training); // Update global default collections. _add_elements_to_collection(_updates.ToArray(), new string[] { ops.GraphKeys.UPDATE_OPS }); diff --git a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj index 6989c08c..d3ca991f 100644 --- a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj +++ b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj @@ -43,7 +43,7 @@ Docs: https://tensorflownet.readthedocs.io - + diff --git a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj index 9e83fdae..503fd643 100644 --- a/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj +++ b/test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj @@ -6,7 +6,7 @@ - + diff --git a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj index c9b02eab..7ddb3a6b 100644 --- a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj @@ -19,7 +19,7 @@ - +