diff --git a/src/TensorFlowNET.Core/APIs/tf.control_flow.cs b/src/TensorFlowNET.Core/APIs/tf.control_flow.cs index 239487e0..578f23f9 100644 --- a/src/TensorFlowNET.Core/APIs/tf.control_flow.cs +++ b/src/TensorFlowNET.Core/APIs/tf.control_flow.cs @@ -57,6 +57,21 @@ namespace Tensorflow new[] { loop_vars }); return results[0]; } + public (Tensor, List, Tensors, Tensors) while_loop(Func cond, + Func, Tensors, Tensors, (Tensor, List, Tensors, Tensors)> body, + (Tensor, List, Tensors, Tensors) loop_vars, + int parallel_iterations = 10) + => control_flow_ops.while_loop(cond, + body, + loop_vars); + + public (Tensor, List, Tensors) while_loop(Func cond, + Func, Tensors, (Tensor, List, Tensors)> body, + (Tensor, List, Tensors) loop_vars, + int parallel_iterations = 10) + => control_flow_ops.while_loop(cond, + body, + loop_vars); public Tensor[] while_loop(Func cond, Func body, diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 116ff7a2..25a98b0c 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -1,17 +1,36 @@ using Newtonsoft.Json; +using OneOf; using System.Collections.Generic; +<<<<<<< HEAD using Tensorflow.Keras.Layers.Rnn; +======= +using Tensorflow.Keras.Layers; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.NumPy; +>>>>>>> master namespace Tensorflow.Keras.ArgsDefinition.Rnn { // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs { +<<<<<<< HEAD [JsonProperty("cell")] // TODO: the cell should be serialized with `serialize_keras_object`. public IRnnCell Cell { get; set; } = null; [JsonProperty("cells")] public IList Cells { get; set; } = null; +======= + public interface IRnnArgCell : ILayer + { + public Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null); + public StateSizeWrapper state_size { get; set; } + public int output_size { get; set; } + } + [JsonProperty("cell")] + // TODO: the cell should be serialized with `serialize_keras_object`. + public OneOf, IRnnArgCell> Cell { get; set; } +>>>>>>> master [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; @@ -26,6 +45,7 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn [JsonProperty("time_major")] public bool TimeMajor { get; set; } = false; // TODO: Add `num_constants` and `zero_output_for_mask`. + public bool ZeroOutputForMask { get; set; } = false; public Dictionary Kwargs { get; set; } = null; public int Units { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs index fcfd694d..d8fdfae5 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs @@ -2,6 +2,9 @@ { public class SimpleRNNArgs : RNNArgs { - + public float Dropout = 0f; + public float RecurrentDropout = 0f; + public int state_size; + public int output_size; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index ea6f830b..e3c18973 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -1,11 +1,19 @@ using System.Collections.Generic; +<<<<<<< HEAD using Tensorflow.Keras.Layers.Rnn; +======= +using static Tensorflow.Keras.ArgsDefinition.Rnn.RNNArgs; +>>>>>>> master namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class StackedRNNCellsArgs : LayerArgs { +<<<<<<< HEAD public IList Cells { get; set; } +======= + public IList Cells { get; set; } +>>>>>>> master public Dictionary Kwargs { get; set; } = null; } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index e94c8bf1..9738052c 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -28,5 +28,11 @@ namespace Tensorflow.Keras TF_DataType DType { get; } int count_params(); void adapt(Tensor data, int? batch_size = null, int? steps = null); + + Tensors Call(Tensors inputs, Tensor? mask = null, bool? training = null, Tensors? initial_state = null, Tensors? constants = null); + + StateSizeWrapper state_size { get; } + + int output_size { get; } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index a19508d4..026a442f 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -226,6 +226,7 @@ namespace Tensorflow.Keras.Layers bool return_sequences = false, bool return_state = false); +<<<<<<< HEAD public ILayer RNN( IRnnCell cell, bool return_sequences = false, @@ -245,6 +246,17 @@ namespace Tensorflow.Keras.Layers bool unroll = false, bool time_major = false ); +======= + public ILayer SimpleRNNCell( + int units, + string activation = "tanh", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f); +>>>>>>> master public ILayer Subtract(); } diff --git a/src/TensorFlowNET.Core/NumPy/StateSizeWrapper.cs b/src/TensorFlowNET.Core/NumPy/StateSizeWrapper.cs new file mode 100644 index 00000000..f2a9e5f2 --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/StateSizeWrapper.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Collections; + + +namespace Tensorflow.NumPy +{ + // Since state_size in RNN is a single integer or array of integer, so use StateSizeWrapper to hold it + public class StateSizeWrapper : IEnumerable + { + int[] _state_size; + public int[] state_size => _state_size; + + public StateSizeWrapper(int state_size) + { + _state_size = new int[] { state_size }; + } + + public StateSizeWrapper(params int[] state_size) + { + _state_size = state_size; + } + public StateSizeWrapper(IEnumerable state_size) + { + _state_size = state_size.ToArray(); + } + + public static implicit operator StateSizeWrapper(int[] state_size) + => new StateSizeWrapper(state_size); + + public static implicit operator StateSizeWrapper(int state_size) + => new StateSizeWrapper(state_size); + + public static implicit operator StateSizeWrapper((int, int) state_size) + => new StateSizeWrapper(state_size.Item1, state_size.Item2); + + public static implicit operator StateSizeWrapper(List v) + => new StateSizeWrapper(v); + public override string ToString() + { + return $"{state_size}"; + } + + public int this[int n] + { + get => n < 0 ? state_size[state_size.Length + n] : state_size[n]; + set => state_size[n] = value; + } + + public IEnumerator GetEnumerator() + { + return state_size.ToList().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} + + diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 26646b76..8daa3de3 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -28,6 +28,7 @@ using Tensorflow.Operations; using Tensorflow.Train; using Tensorflow.Util; using static Tensorflow.Binding; +using static Tensorflow.Keras.ArgsDefinition.Rnn.RNNArgs; namespace Tensorflow { @@ -52,8 +53,12 @@ namespace Tensorflow /// matching structure of Tensors having shape `[batch_size].concatenate(s)` /// for each `s` in `self.batch_size`. /// +<<<<<<< HEAD [Obsolete("This is an incompleted tf v1 api, pleas use keras RNNs instead.")] public abstract class RnnCell : ILayer, IRnnCell +======= + public abstract class RnnCell : ILayer +>>>>>>> master { /// /// Attribute that indicates whether the cell is a TF RNN cell, due the slight @@ -91,6 +96,8 @@ namespace Tensorflow protected bool built = false; public bool Built => built; + StateSizeWrapper ILayer.state_size => throw new NotImplementedException(); + public RnnCell(bool trainable = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid, @@ -177,6 +184,7 @@ namespace Tensorflow throw new NotImplementedException(); } +<<<<<<< HEAD public (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) { throw new NotImplementedException(); @@ -185,5 +193,11 @@ namespace Tensorflow public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); public bool IsTFRnnCell => throw new NotImplementedException(); public bool SupportOptionalArgs => throw new NotImplementedException(); +======= + public Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { + throw new NotImplementedException(); + } +>>>>>>> master } } diff --git a/src/TensorFlowNET.Core/Operations/control_flow_ops.cs b/src/TensorFlowNET.Core/Operations/control_flow_ops.cs index 862b636f..c59e5b16 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_ops.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_ops.cs @@ -698,6 +698,53 @@ namespace Tensorflow }); } + public static (Tensor, List, Tensors, Tensors) while_loop(Func cond, + Func, Tensors, Tensors, (Tensor, List, Tensors, Tensors)> body, + (Tensor, List, Tensors, Tensors) loop_vars, + int parallel_iterations = 10, + string name = null) + { + var executing_eagerly = tf.Context.executing_eagerly(); + if (!executing_eagerly) + { + throw new NotImplementedException(""); + } + + return tf_with(ops.name_scope("name", "while"), delegate + { + while ((bool)cond(loop_vars.Item1)) + { + loop_vars = body(loop_vars.Item1, loop_vars.Item2, loop_vars.Item3, loop_vars.Item4); + } + + return loop_vars; + }); + } + + public static (Tensor, List, Tensors) while_loop(Func cond, + Func, Tensors, (Tensor, List, Tensors)> body, + (Tensor, List, Tensors) loop_vars, + int parallel_iterations = 10, + string name = null) + { + var executing_eagerly = tf.Context.executing_eagerly(); + if (!executing_eagerly) + { + throw new NotImplementedException(""); + } + + return tf_with(ops.name_scope("name", "while"), delegate + { + while ((bool)cond(loop_vars.Item1)) + { + loop_vars = body(loop_vars.Item1, loop_vars.Item2, loop_vars.Item3); + } + + return loop_vars; + }); + } + + /// /// Repeat `body` while the condition `cond` is true. /// diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index 3456d9b3..68d561ae 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -4633,8 +4633,9 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatMul", name) { args = new object[] { a, b }, attrs = new Dictionary() { ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b } }); return _fast_path_result[0]; } - catch (Exception) + catch (ArgumentException) { + throw new ArgumentException("In[0] and In[1] has diffrent ndims!"); } try { diff --git a/src/TensorFlowNET.Core/Util/nest.py.cs b/src/TensorFlowNET.Core/Util/nest.py.cs index 3ba3ce78..b620f0ba 100644 --- a/src/TensorFlowNET.Core/Util/nest.py.cs +++ b/src/TensorFlowNET.Core/Util/nest.py.cs @@ -19,6 +19,7 @@ using System; using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; namespace Tensorflow.Util { @@ -212,6 +213,39 @@ namespace Tensorflow.Util => arg is IEnumerable && !(arg is string) && !(arg is NDArray) && !(arg.GetType().IsGenericType && arg.GetType().GetGenericTypeDefinition() == typeof(HashSet<>)); + public static bool is_nested(object obj) + { + // Refer to https://www.tensorflow.org/api_docs/python/tf/nest + //if (obj is IList || obj is IDictionary || obj is ITuple) + // return true; + if (obj is IList || obj is IDictionary) + return true; + + if (obj is NDArray || obj is Tensor || obj is string || obj.GetType().IsGenericType + || obj is ISet || obj is ISet || obj is ISet) + return false; + + if (obj.GetType().IsNested) return true; + // Check if the object is an IEnumerable + if (obj is IEnumerable) + { + // If it is, check if it is a nested structure + foreach (object item in (IEnumerable)obj) + { + if (is_nested(item)) + { + return true; + } + } + return true; + } + else + { + // If it is not, return false + return false; + } + } + public static bool is_mapping(object arg) => arg is IDictionary; //# See the swig file (util.i) for documentation. @@ -223,7 +257,13 @@ namespace Tensorflow.Util _flatten_recursive(structure, list); return list; } - + // TODO(Wanglongzhi2001), ITuple must used in .NET standard 2.1, but now is 2.0 + // If you want to flatten a nested tuple, please specify the type of the tuple + //public static List flatten(ITuple structure) + //{ + // var list = FlattenTuple(structure).ToList(); + // return list; + //} public static List flatten(IEnumerable structure) { var list = new List(); @@ -251,9 +291,13 @@ namespace Tensorflow.Util case String str: list.Add(obj); break; - case NDArray nd: + // This case can hold both Tensor and NDArray + case Tensor tensor: list.Add(obj); break; + //case NDArray nd: + // list.Add(obj); + // break; case IEnumerable structure: foreach (var child in structure) _flatten_recursive((T)child, list); @@ -264,7 +308,27 @@ namespace Tensorflow.Util } } + private static IEnumerable FlattenTuple(object tuple) + { + //if (tuple is ITuple t) + //{ + // for (int i = 0; i < t.Length; i++) + // { + // foreach (var item in FlattenTuple(t[i])) + // { + // yield return item; + // } + // } + //} + if(false) + { + } + else + { + yield return (T)tuple; + } + } //# See the swig file (util.i) for documentation. //_same_namedtuples = _pywrap_tensorflow.SameNamedtuples @@ -451,8 +515,12 @@ namespace Tensorflow.Util throw new ArgumentException("flat_sequence must not be null"); // if not is_sequence(flat_sequence): // raise TypeError("flat_sequence must be a sequence") - - if (!is_sequence(structure)) + if (!is_nested(flat_sequence)) + { + throw new ArrayTypeMismatchException($"Attempted to pack value:\\n {flat_sequence}\\ninto a structure, " + + $"but found incompatible type `{flat_sequence.GetType()}` instead."); + } + if (!is_nested(structure)) { if (len(flat) != 1) throw new ValueError($"Structure is a scalar but len(flat_sequence) == {len(flat)} > 1"); diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 1336e9af..1021a6c9 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -24,7 +24,12 @@ using Tensorflow.Common.Extensions; using static Tensorflow.Binding; using static Tensorflow.Graphs.SubGraphUtility; using Tensorflow.Util; +<<<<<<< HEAD using Tensorflow.Common.Types; +======= +using Tensorflow.Operations; +using OneOf; +>>>>>>> master namespace Tensorflow.Keras { @@ -68,7 +73,7 @@ namespace Tensorflow.Keras return; } var graph = v.Graph; - if(graph is null) + if (graph is null) { graph = get_graph(); } @@ -98,7 +103,7 @@ namespace Tensorflow.Keras { if (_GRAPH == null) _GRAPH = new FuncGraph("keras_graph"); - + return _GRAPH; } return ops.get_default_graph(); @@ -108,7 +113,7 @@ namespace Tensorflow.Keras { if (_CURRENT_SCRATCH_GRAPH == null) _CURRENT_SCRATCH_GRAPH = new FuncGraph("keras_scratch_graph"); - + return _CURRENT_SCRATCH_GRAPH; } @@ -233,16 +238,16 @@ namespace Tensorflow.Keras { if (outputs[0].op.type == "Const") return tensor_util.constant_value(outputs); - + var source_graph = outputs.graph; var exec_graph = _scratch_graph(); var global_graph = get_graph(); if (source_graph == global_graph && exec_graph != global_graph) { - var lifted_map = lift_to_graph(outputs, exec_graph, - new List(), - add_sources: true, - handle_captures: true, + var lifted_map = lift_to_graph(outputs, exec_graph, + new List(), + add_sources: true, + handle_captures: true, base_graph: source_graph); } if (outputs[0].op.type == "Placeholder" @@ -253,7 +258,7 @@ namespace Tensorflow.Keras exec_graph.as_default(); exec_graph.Inputs = exec_graph.internal_captures; exec_graph.Outputs = outputs; - + var graph_fn = new ConcreteFunction(exec_graph); _CURRENT_SCRATCH_GRAPH = null; @@ -373,7 +378,7 @@ namespace Tensorflow.Keras /// /// /// - public Tensor resize_images(Tensor x, int height_factor, int width_factor, + public Tensor resize_images(Tensor x, int height_factor, int width_factor, string data_format, string interpolation = "nearest") { var (rows, cols) = (0, 0); @@ -415,7 +420,7 @@ namespace Tensorflow.Keras /// public Tensor concatenate(Tensors tensors, int axis = -1) { - if(axis < 0) + if (axis < 0) { var rank = tensors[0].ndim; if (rank > -1) @@ -454,6 +459,7 @@ namespace Tensorflow.Keras return x; } +<<<<<<< HEAD public (Tensors, Tensors, Tensors) rnn( Func step_function, // args:inputs, states, return:output, new_states Tensors inputs, // inputs is a tuple of tensors (one per input sequence) @@ -469,6 +475,29 @@ namespace Tensorflow.Keras { Tensor swap_batch_timestep(Tensor input_t) +======= + public static (Tensors, Tensors) convert_inputs_if_ragged(OneOf inputs) + { + throw new NotImplementedException(); + } + + // + public static (Tensors, Tensors, Tensors) rnn( + Func step_function, // args:inputs, states, return:output, new_states + Tensors inputs, // inputs is a tuple of tensors (one per input sequence) + Tensors initial_states, + bool go_backwards = false, + Tensor? mask = null, + Tensors? constants = null, + bool unroll = false, + Tensors? input_length = null, // An integer or a 1-D Tensor,depending on whether the time dimension is fixed-length or not + bool time_major = false, + bool zero_output_for_mask = false, + bool return_all_outputs = true) + { + + Tensors swap_batch_timestep(Tensors input_t) +>>>>>>> master { var axes = Enumerable.Range(0, input_t.rank).ToArray(); axes[0] = 1; @@ -478,6 +507,7 @@ namespace Tensorflow.Keras if (!time_major) { +<<<<<<< HEAD inputs = Nest.MapStructure(swap_batch_timestep, inputs).ToTensors(); } @@ -486,6 +516,15 @@ namespace Tensorflow.Keras var time_steps = first_flatted_input.shape[0]; var batch = first_flatted_input.shape[1]; var time_steps_t = (int)first_flatted_input.shape[0]; +======= + inputs = nest.map_structure(swap_batch_timestep, inputs); + } + + var flatted_inptus = nest.flatten(inputs); + var time_steps = flatted_inptus[0].shape[0]; + var batch = flatted_inptus[0].shape[1]; + var time_step_t = tf.shape(flatted_inptus[0])[0]; +>>>>>>> master foreach (var input_ in flatted_inptus) { @@ -510,7 +549,16 @@ namespace Tensorflow.Keras } } +<<<<<<< HEAD +======= + + if (constants == null) + { + constants = new List(); + } + +>>>>>>> master // tf.where needs its condition tensor to be the same shape as its two // result tensors, but in our case the condition (mask) tensor is // (nsamples, 1), and inputs are (nsamples, ndimensions) or even more. @@ -520,12 +568,20 @@ namespace Tensorflow.Keras Tensors _expand_mask(Tensors mask_t, Tensors input_t, int fixed_dim = 1) { +<<<<<<< HEAD if (!mask_t.IsSingle()) +======= + if (nest.is_nested(mask_t)) +>>>>>>> master { throw new ValueError($"mask_t is expected to be tensor, but got {mask_t}"); } +<<<<<<< HEAD if (!input_t.IsSingle()) +======= + if (nest.is_nested(input_t)) +>>>>>>> master { throw new ValueError($"input_t is expected to be tensor, but got {input_t}"); } @@ -535,7 +591,11 @@ namespace Tensorflow.Keras { mask_t = tf.expand_dims(mask_t, -1); } +<<<<<<< HEAD var multiples = Enumerable.Repeat(1, fixed_dim).ToArray().concat(input_t.shape.as_int_list().Skip(fixed_dim).ToArray()); +======= + var multiples = Enumerable.Repeat(1, fixed_dim).ToArray().concat(input_t.shape.as_int_list().ToList().GetRange(fixed_dim, input_t.rank)); +>>>>>>> master return tf.tile(mask_t, multiples); } @@ -570,6 +630,7 @@ namespace Tensorflow.Keras // individually. The result of this will be a tuple of lists, each of // the item in tuple is list of the tensor with shape (batch, feature) +<<<<<<< HEAD Tensors _process_single_input_t(Tensor input_t) { var unstaked_input_t = array_ops.unstack(input_t); // unstack for time_step dim @@ -578,13 +639,32 @@ namespace Tensorflow.Keras unstaked_input_t = unstaked_input_t.Reverse().ToArray(); } return unstaked_input_t; +======= + + + + Tensors _process_single_input_t(Tensors input_t) + { + input_t = tf.unstack(input_t); // unstack for time_step dim + if (go_backwards) + { + input_t.Reverse(); + } + return input_t; +>>>>>>> master } // TODO(Wanglongzhi2001) Tensors processed_input; +<<<<<<< HEAD if (!inputs.IsSingle()) { processed_input = inputs.MapStructure(_process_single_input_t).ReduceTo().ToTensors(); +======= + if (nest.is_nested(inputs)) + { + processed_input = nest.map_structure(_process_single_input_t, inputs); +>>>>>>> master } else { @@ -598,6 +678,7 @@ namespace Tensorflow.Keras { inp.Add(t_[time]); } +<<<<<<< HEAD return Nest.PackSequenceAs(inputs, inp); } @@ -925,6 +1006,336 @@ namespace Tensorflow.Keras last_output = Nest.PackSequenceAs(output_time_zero, last_output).ToTensors(); } +======= + return nest.pack_sequence_as(inputs, inp); + } + + //if (mask != null) + //{ + // var mask_list = tf.unstack(mask); + // if (go_backwards) + // { + // mask_list.Reverse(); + // } + + // for (int i = 0; i < time_steps; i++) + // { + // // TODO(Wanglongzhi2001),deal with _get_input_tensor + // var inp = _get_input_tensor(i); + // var mask_t = mask_list[i]; + // // TODO + // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); + + // var tiled_mask_t = _expand_mask(mask_t, output); + + // Tensors prev_output; + // if (successive_outputs == null) + // { + // prev_output = tf.zeros_like(output); + // } + // else + // { + // prev_output = successive_outputs[successive_outputs.Length - 1]; + // } + + // output = tf.where(tiled_mask_t, output, prev_output); + + // //var flat_states = nest.flatten(states); + // //var flat_new_states = nest.flatten(newStates); + // var flat_states = states.ToList(); + // var flat_new_states = newStates.ToList(); + + // var tiledMaskT = flat_states + // .Select(s => _expand_mask(mask_t, s)) + // .ToArray(); + // var tuple = Tuple.Create(tiledMaskT); + + // List flat_final_states = new List(); + // foreach (var (m, s, ps) in Enumerable.Zip(tiled_mask_t, flat_new_states, flat_states)) + // { + // flat_final_states.Add(tf.where(m, s, ps)); + // } + + // states = (Tensors)nest.pack_sequence_as(states, flat_final_states); + // if (return_all_outputs) + // { + // successive_outputs.Add(output); + // successive_states.Add(states); + // } + // else + // { + // successive_outputs = new Tensors { output }; + // successive_states = new Tensors { states }; + // } + + // } + // last_output = successive_outputs[successive_outputs.Length - 1]; + // new_states = successive_states[successive_states.Length - 1]; + // outputs = tf.stack(successive_outputs); + + // if (zero_output_for_mask) + // { + // last_output = tf.where(_expand_mask(mask_list[mask_list.Length - 1], last_output), last_output, tf.zeros_like(last_output)); + // outputs = tf.where(_expand_mask(mask, outputs, fixed_dim: 2), outputs, tf.zeros_like(outputs)); + // } + // else // mask is null + // { + // for (int i = 0; i < time_steps; i++) + // { + // var inp = _get_input_tensor(i); + // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); + // states = newStates; + + // if (return_all_outputs) + // { + // successive_outputs.Add(output); + // successive_states.Add(newStates); + // } + // else + // { + // successive_outputs = new Tensors { output }; + // successive_states = new Tensors { newStates }; + // } + // } + // last_output = successive_outputs[successive_outputs.Length - 1]; + // new_states = successive_states[successive_states.Length - 1]; + // outputs = tf.stack(successive_outputs); + // } + //} + } + //else // unroll == false + //{ + // var states = initial_states; + // // Create input tensor array, if the inputs is nested tensors, then it + // // will be flattened first, and tensor array will be created one per + // // flattened tensor. + // var input_ta = new List(); + // for (int i = 0; i < flatted_inptus.Count; i++) + // { + // input_ta.Add(tf.TensorArray(dtype: flatted_inptus[i].dtype, size: time_step_t)); + // } + + // // Get the time(0) input and compute the output for that, the output will + // // be used to determine the dtype of output tensor array. Don't read from + // // input_ta due to TensorArray clear_after_read default to True. + // var inps = new Tensors(); + // foreach (var inp in flatted_inptus) + // { + // inps.Add(inp[0]); + // } + // var input_time_zero = nest.pack_sequence_as(inputs, inps); + + // // output_time_zero is used to determine the cell output shape and its + // // dtype. the value is discarded. + // (output_time_zero, _) = step_function((Tensor)input_time_zero, new Tensors { initial_states, constants }); + + // var output_ta_size = return_all_outputs ? time_step_t : tf.constant(1); + // var output_ta = new List(); + // for (int i = 0; i < output_time_zero.ToList().Count; i++) + // { + // var Out = output_time_zero.ToList()[i]; + // output_ta.Add(tf.TensorArray(dtype: Out.dtype, size: output_ta_size, element_shape: Out.shape)); + // } + + // var time = tf.constant(0, dtype: TF_DataType.TF_INT32, name: "time"); + + + + // Func? masking_fn; + // Func? compute_masked_output = null; + // if (mask != null) + // { + // if (go_backwards) + // { + // mask = tf.reverse(mask, axis: new[] { 0 }); + // } + // var mask_ta = tf.TensorArray(dtype: TF_DataType.TF_BOOL, size: time_step_t); + // mask_ta = mask_ta.unstack(mask); + + // masking_fn = (time) => + // { + // return mask_ta.read(time); + // }; + + // compute_masked_output = (mask_t, flat_out, flat_mask) => + // { + // var tiled_mask_t = new Tensors(); + // foreach (var o in flat_out) + // { + // tiled_mask_t.Add(_expand_mask(mask_t, o, fixed_dim: mask_t.rank)); + // } + + // Tensors res = new Tensors(); + // foreach (var (m, o, fm) in Enumerable.Zip(tiled_mask_t, flat_out, flat_mask)) + // { + // res.Add(tf.where(m, o, fm)); + // } + // return res; + // }; + // } + // // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor)? + // else if (input_length is Tensor) + // { + // if (go_backwards) + // { + // var max_len = tf.reduce_max(input_length, axis: 0); + // var rev_input_length = tf.subtract(max_len - 1, input_length); + + // masking_fn = (time) => + // { + // return tf.less(rev_input_length, time); + // }; + // } + // else + // { + // masking_fn = (time) => + // { + // return tf.greater(input_length, time); + // }; + // } + + // compute_masked_output = (mask_t, flat_out, flat_mask) => + // { + // var res = new List(); + // foreach (var (o, zo) in zip(flat_out, flat_mask)) + // { + // res.Add(tf.where(mask_t, o, zo)); + // } + // return res; + // }; + // } + // else + // { + // masking_fn = null; + // } + + + // if (masking_fn != null) + // { + // // Mask for the T output will be base on the output of T - 1. In the + // // case T = 0, a zero filled tensor will be used. + // var flat_zero_output = new Tensors(); + // foreach (var o in nest.flatten(output_time_zero)) + // { + // flat_zero_output.Add(tf.zeros_like(o)); + // } + + + // (Tensor, List, Tensors, Tensors) _step(Tensor time, List output_ta_t, Tensors prev_output, Tensors states) + // { + // /* + // RNN step function. + // Args: + // time: Current timestep value. + // output_ta_t: TensorArray. + // prev_output: tuple of outputs from time - 1. + // *states: List of states. + // Returns: + // Tuple(todo): `(time + 1, output_ta_t, output) + tuple(new_states)` + // */ + + // var current_input = input_ta.Select(x => x.read(time)).ToList(); + // // maybe set shape + // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + // current_input = (List)nest.pack_sequence_as(inputs, current_input); + // var mask_t = masking_fn(time); + // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); + // // mask output + // //var flat_output = nest.flatten(output); + // var flat_output = output.ToList(); + + // var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.ToList(); + + // // TODO(Wanglongzhi2001),deal with compute_masked_output's third parameter's type + // var flat_new_output = compute_masked_output(mask_t, flat_output, flat_mask_output); + + // // mask states + // var flat_state = states.ToList(); + // var flat_new_state = new_states.ToList(); + + // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + // { + // if (new_state is Tensor) + // { + // new_state.set_shape(state.shape); + // } + // } + + // var flat_final_state = compute_masked_output(mask_t, flat_new_state, flat_state); + // new_states = (Tensors)nest.pack_sequence_as(new_states, flat_final_state); + + // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + // var Output_ta_t = new List(); + // // TODO(Wanglongzhi2001),deal with zip output_ta_t + // foreach (var (ta, Out) in zip(output_ta_t, flat_new_output)) + // { + // Output_ta_t.Add(ta.write(ta_index_to_write, Out)); + // } + + + + // //new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); + + + // return (time + 1, Output_ta_t, flat_new_output, new_states); + + // } + // Func cond = (time) => (time < time_step_t); + + // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, flat_zero_output, states)); + // new_states = final_outputs.Item4; + // output_ta = final_outputs.Item2; + + // } + // else + // { + // (Tensor, List, Tensors) _step(Tensor time, List output_ta_t, Tensors states) + // { + // var current_input = input_ta.Select(x => x.read(time)).ToList(); + // // maybe set shape + // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + // current_input = (List)nest.pack_sequence_as(inputs, current_input); + // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); + // var flat_state = states.ToList(); + // var flat_new_state = new_states.ToList(); + // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + // { + // if (new_state is Tensor) + // { + // new_state.set_shape(state.shape); + // } + // } + // var flat_output = output.ToList(); + // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + // var Output_ta_t = new List(); + // foreach (var (ta, out_) in zip(output_ta_t, flat_output)) + // { + // Output_ta_t.Add(ta.write(ta_index_to_write, out_)); + // } + + // new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); + // return (time + 1, Output_ta_t, new_states); + // } + // Func cond = (time) => (time < time_step_t); + // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, states)); + // new_states = final_outputs.Item3; + // output_ta = final_outputs.Item2; + + // } + // //Tensors outputs = new Tensors(); + // foreach (var o in output_ta) + // { + // outputs.Add(o.stack()); + // } + // foreach (var o in outputs) + // { + // last_output.Add(o[-1]); + // } + // outputs = (Tensors)nest.pack_sequence_as(output_time_zero, outputs); + // last_output = (Tensors)nest.pack_sequence_as(output_time_zero, last_output); + + //} +>>>>>>> master Func set_shape; set_shape = (output_) => @@ -941,11 +1352,16 @@ namespace Tensorflow.Keras shape[0] = 1; } shape[1] = (int)batch; +<<<<<<< HEAD output_.shape = shape; +======= + output_.set_shape(new Tensor(shape)); +>>>>>>> master } return output_; }; +<<<<<<< HEAD outputs = Nest.MapStructure(set_shape, outputs).ToTensors(); if (!time_major) { @@ -973,6 +1389,37 @@ namespace Tensorflow.Keras } throw new NotImplementedException("Not implemented currently, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); +======= + var Outputs = (Tensors)nest.map_structure(set_shape, outputs); + if (!time_major) + { + Outputs = nest.map_structure(swap_batch_timestep, outputs); + } + return (last_output, Outputs, new_states); + + } + + // Multiplies 2 tensors (and/or variables) and returns a tensor. + // This operation corresponds to `numpy.dot(a, b, out=None)`. + public Tensor Dot(Tensor x, Tensor y) + { + //if (x.ndim != 1 && (x.ndim > 2 || y.ndim > 2)) + //{ + // var x_shape = new List(); + // foreach (var (i,s) in zip(x.shape.as_int_list(), tf.unstack(tf.shape(x)))) + // { + // if (i != 0) + // { + // x_shape.append(i); + // } + // else + // { + // x_shape.append(s); + // } + // } + //} + throw new NotImplementedException(); +>>>>>>> master } } } diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 7347585f..1f8f027f 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -326,7 +326,11 @@ namespace Tensorflow.Keras.Engine nodes_in_decreasing_depth.append(node); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var tensor_dict = new Dictionary>(); // map input values diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index a0358f07..72ac80a0 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -31,7 +31,11 @@ namespace Tensorflow.Keras.Engine if (!built) MaybeBuild(inputs); +<<<<<<< HEAD var outputs = Call(inputs, state: states, training: training); +======= + var outputs = Call(inputs, initial_state: state, training: training); +>>>>>>> master // memory leak // _set_connectivity_metadata_(inputs, outputs); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index 2f758a85..99b040ad 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -254,6 +254,10 @@ namespace Tensorflow.Keras.Engine /// public Func? ReplacedCall { get; set; } = null; + public StateSizeWrapper state_size => throw new NotImplementedException(); + + public int output_size => throw new NotImplementedException(); + public Layer(LayerArgs args) { Initialize(args); @@ -332,9 +336,13 @@ namespace Tensorflow.Keras.Engine /// /// /// +<<<<<<< HEAD protected virtual Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected virtual Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { - if(ReplacedCall is not null) + if (ReplacedCall is not null) { return ReplacedCall(inputs); } @@ -434,56 +442,61 @@ namespace Tensorflow.Keras.Engine public override void SetAttr(string name, object value) { - // TODO(Rinne): deal with "_self_setattr_tracking". + //// TODO(Rinne): deal with "_self_setattr_tracking". - value = TrackableDataStructure.sticky_attribute_assignment(this, name, value); + //value = TrackableDataStructure.sticky_attribute_assignment(this, name, value); - foreach(var val in nest.flatten(value)) - { - if(val is Metric) - { - // TODO(Rinne): deal with metrics. - } - } - - // TODO(Rinne): deal with "_auto_track_sub_layers". - - foreach(var val in nest.flatten(value)) - { - if(val is not IVariableV1 variable) - { - continue; - } - if (variable.Trainable) - { - if (_trainable_weights.Contains(variable)) - { - continue; - } - _trainable_weights.Add(variable); - } - else - { - if (_non_trainable_weights.Contains(variable)) - { - continue; - } - _non_trainable_weights.Add(variable); - } - keras.backend.track_variable(variable); - } + //foreach(var val in nest.flatten(value)) + //{ + // if(val is Metric) + // { + // // TODO(Rinne): deal with metrics. + // } + //} + + //// TODO(Rinne): deal with "_auto_track_sub_layers". + + //foreach(var val in nest.flatten(value)) + //{ + // if(val is not IVariableV1 variable) + // { + // continue; + // } + // if (variable.Trainable) + // { + // if (_trainable_weights.Contains(variable)) + // { + // continue; + // } + // _trainable_weights.Add(variable); + // } + // else + // { + // if (_non_trainable_weights.Contains(variable)) + // { + // continue; + // } + // _non_trainable_weights.Add(variable); + // } + // keras.backend.track_variable(variable); + //} + + //// Directly use the implementation of `Trackable`. + //var t = this.GetType(); + //var field_info = t.GetField(name); + //if (field_info is not null) + //{ + // field_info.SetValue(this, value); + //} + //else + //{ + // CustomizedFields[name] = value; + //} + } - // Directly use the implementation of `Trackable`. - var t = this.GetType(); - var field_info = t.GetField(name); - if (field_info is not null) - { - field_info.SetValue(this, value); - } - else - { - CustomizedFields[name] = value; - } + Tensors ILayer.Call(Tensors inputs, Tensor mask, bool? training, Tensors initial_state, Tensors constants) + { + throw new NotImplementedException(); } } } diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 6a468ad2..e5a8211b 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -144,7 +144,11 @@ namespace Tensorflow.Keras.Engine } } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (!_has_explicit_input_shape) { @@ -155,10 +159,10 @@ namespace Tensorflow.Keras.Engine { if (!built) _init_graph_network(this.inputs, outputs); - return base.Call(inputs, state, training); + return base.Call(inputs, initial_state, training); } - return base.Call(inputs, state, training); + return base.Call(inputs, initial_state, training); } void _build_graph_network_for_inferred_shape(Shape input_shape, TF_DataType input_dtype) diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs index 23f36c86..bf8790da 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs @@ -30,7 +30,11 @@ namespace Tensorflow.Keras.Layers { base.build(input_shape); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor output = inputs; output = tf.where(output > 0f, output, diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs index 81fefb31..cddf0296 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs @@ -17,7 +17,11 @@ namespace Tensorflow.Keras.Layers { { base.build(input_shape); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor output = inputs; return tf.exp(output); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs b/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs index e0f91380..9fc62bb2 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs @@ -11,7 +11,12 @@ namespace Tensorflow.Keras.Layers { public HardSigmoid ( LayerArgs args ) : base(args) { // hard sigmoid has no arguments } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null ) { +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { +>>>>>>> master Tensor x = inputs; return tf.clip_by_value( tf.add(tf.multiply(x, 0.2f), 0.5f), 0f, 1f); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs b/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs index cfbd0186..fa2d2b6a 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs @@ -20,7 +20,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { return tf.nn.leaky_relu(inputs, alpha: alpha); } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs index 2e943d5f..ddd7595b 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs @@ -23,7 +23,12 @@ namespace Tensorflow.Keras.Layers { } base.build(input_shape); } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { +>>>>>>> master Tensor output = inputs; return tf.where(output > 0f, tf.multiply(scale, output), diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs index d018128d..535a8213 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs @@ -12,8 +12,14 @@ namespace Tensorflow.Keras.Layers { public Softmax ( SoftmaxArgs args ) : base(args) { axis = args.axis; } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor x = inputs.Length == 2 ? inputs[0] + ((1.0 - tf.cast(inputs[1], inputs.dtype)) * 1e-9) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { + Tensor x = inputs.Length == 2 ? inputs + ((1.0 - tf.cast(inputs[1], inputs.dtype)) * 1e-9) +>>>>>>> master : inputs; Tensor e = tf.exp(tf.sub(x, tf.reduce_max(x, axis: this.axis, keepdims: true))); Tensor s = tf.reduce_sum(e, axis: this.axis, keepdims: true); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs index 1e6c59b4..17a43475 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs @@ -11,7 +11,12 @@ namespace Tensorflow.Keras.Layers { public Softplus ( LayerArgs args ) : base(args) { // Softplus has no arguments } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { +>>>>>>> master Tensor x = inputs; return tf.log( tf.add(tf.exp(x), 1f)); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs index 5ad33e99..c16570f7 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs @@ -11,7 +11,12 @@ namespace Tensorflow.Keras.Layers { public Softsign ( LayerArgs args ) : base(args) { // Softsign has no arguments } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { +>>>>>>> master Tensor x = inputs; // x / (abs(x) + 1) return tf.div(x, tf.add(1f, tf.abs(x))); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs b/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs index ed0d105a..6da3e1ca 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs @@ -11,7 +11,12 @@ namespace Tensorflow.Keras.Layers { public Swish ( LayerArgs args ) : base(args) { // Swish has no arguments } +<<<<<<< HEAD protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { +>>>>>>> master Tensor x = inputs; // x / (1 + exp(-x)) diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs b/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs index 7e90cf9d..8ecd7bf2 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs @@ -14,7 +14,11 @@ namespace Tensorflow.Keras.Layers { // Tanh has no arguments } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor x = inputs; diff --git a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs index 19b29272..ff7b2092 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs @@ -115,7 +115,11 @@ namespace Tensorflow.Keras.Layers return (tf.linalg.einsum("bij,bjk->bik", (weights, value)), weights); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensors _inp; Tensors _mask = null; diff --git a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs index 75dd4a41..98efd1bb 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs @@ -253,7 +253,11 @@ namespace Tensorflow.Keras.Layers return (attention_output, attention_scores); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensors _inp; Tensor _mask = null; diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs index 94ad7914..a45124f9 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs @@ -84,7 +84,11 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var inputs_shape = array_ops.shape(inputs); var batch_size = inputs_shape[0]; diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs index d8e00d52..494c3f85 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs @@ -104,7 +104,11 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = false, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var outputs = _convolution_op.Apply(inputs, kernel.AsTensor()); if (use_bias) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index db5d626e..eff405f7 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -70,7 +70,11 @@ namespace Tensorflow.Keras.Layers built = true; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor outputs = null; var rank = inputs.rank; diff --git a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs index 0cbd5084..d507f2c3 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs @@ -190,7 +190,11 @@ namespace Tensorflow.Keras.Layers // return new dict(base_config.items().ToList() + config.items().ToList()); //} +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var ret = tf.linalg.einsum(this.equation, (inputs, this.kernel.AsTensor())); if (this.bias != null) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs index 87b42bb7..89da542a 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs @@ -67,7 +67,11 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var dtype = inputs.dtype; if (dtype != tf.int32 && dtype != tf.int64) diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 66c3cdc1..d4c93044 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -738,6 +738,27 @@ namespace Tensorflow.Keras.Layers ReturnState = return_state }); + public ILayer SimpleRNNCell( + int units, + string activation = "tanh", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f) + => new SimpleRNNCell(new SimpleRNNArgs + { + Units = units, + Activation = keras.activations.GetActivationFromName(activation), + UseBias = use_bias, + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + Dropout = dropout, + RecurrentDropout = recurrent_dropout + } + ); + /// /// /// diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs index bcbb20d8..b9142df1 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs @@ -22,7 +22,11 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { return _merge_function(inputs); } diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs index 65558157..2819dc33 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs @@ -147,7 +147,11 @@ namespace Tensorflow.Keras.Layers return false; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor outputs = null; var training_tensor = training == null diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs index 1898f24c..cafc2aec 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs @@ -102,7 +102,11 @@ namespace Tensorflow.Keras.Layers return input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor outputs = null; var inputs_dtype = inputs.dtype.as_base_dtype(); diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs index 987b56bc..6ccbc4e0 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs @@ -158,7 +158,11 @@ namespace Tensorflow.Keras.Layers base.adapt(data, batch_size: batch_size, steps: steps); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (_args.Invert) { diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs index ffaabec9..e005e476 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs @@ -13,7 +13,11 @@ namespace Tensorflow.Keras.Layers { } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (data_format == "channels_last") return math_ops.reduce_mean(inputs, 1, false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs index e0666517..f8e5c9e2 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs @@ -13,7 +13,11 @@ namespace Tensorflow.Keras.Layers { } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (data_format == "channels_last") return math_ops.reduce_mean(inputs, (1, 2), false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs index 15695e8a..8badfb53 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs @@ -13,7 +13,11 @@ namespace Tensorflow.Keras.Layers { } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (data_format == "channels_last") return math_ops.reduce_max(inputs, 1, false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs index 76db858d..8a6abc76 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs @@ -13,7 +13,11 @@ namespace Tensorflow.Keras.Layers { } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (data_format == "channels_last") return math_ops.reduce_max(inputs, (1, 2), false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs index 81a34019..c24b4b90 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs @@ -37,7 +37,11 @@ namespace Tensorflow.Keras.Layers input_spec = new InputSpec(ndim: 3); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { int pad_axis = args.DataFormat == "channels_first" ? 2 : 3; inputs = tf.expand_dims(inputs, pad_axis); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs index f83f1e15..cfeba9df 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs @@ -37,7 +37,11 @@ namespace Tensorflow.Keras.Layers input_spec = new InputSpec(ndim: 4); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { int[] pool_shape; int[] strides; diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs index 20d2a53d..f595c538 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs @@ -15,7 +15,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var depth = args.NumTokens; var max_value = tf.reduce_max(inputs); diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs index 7fa367ee..fad07ad7 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs @@ -18,7 +18,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { scale = constant_op.constant(args.Scale, args.DType); offset = constant_op.constant(args.Offset, args.DType); diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs index 081966ad..e1fe611c 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs @@ -20,7 +20,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { return image_ops_impl.resize_images_v2(inputs, new[] { args.Height, args.Width }, method: args.Interpolation); } diff --git a/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs b/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs index ada1851c..cc0d756d 100644 --- a/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs +++ b/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs @@ -16,7 +16,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (training == null) training = false; diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs index 31285438..b20fd5f5 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs @@ -29,7 +29,11 @@ namespace Tensorflow.Keras.Layers.Reshaping _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor output = inputs; if (output.rank != 3) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs index 4a5c6eab..9d835322 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs @@ -22,7 +22,11 @@ namespace Tensorflow.Keras.Layers.Reshaping built = true; _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor output = inputs; if (output.rank != 4) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs index 83f86c6f..d7eafbbb 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs @@ -22,7 +22,11 @@ namespace Tensorflow.Keras.Layers.Reshaping _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor output = inputs; if (output.rank != 5) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs index a6192849..618123ef 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs @@ -24,7 +24,11 @@ namespace Tensorflow.Keras.Layers _channels_first = args.DataFormat == "channels_first"; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (_channels_first) { diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs index 7fdb816b..a1307d48 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs @@ -29,7 +29,11 @@ namespace Tensorflow.Keras.Layers { built = true; _buildInputShape = input_shape; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { Tensor outputs = inputs; return tf.transpose(outputs, new Axis(permute)); diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs index 4b3d30e2..5609bd87 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs @@ -20,7 +20,11 @@ namespace Tensorflow.Keras.Layers this.args = args; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { var shapes = new List(); shapes.Add(array_ops.shape(inputs)[0]); diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs index 223f33d4..537e93c9 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs @@ -25,7 +25,11 @@ namespace Tensorflow.Keras.Layers inputSpec = new InputSpec(ndim: 4); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { return keras.backend.resize_images(inputs, size[0], size[1], diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs index 3b37dac4..b0b409a7 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs @@ -27,7 +27,11 @@ namespace Tensorflow.Keras.Layers this.input_spec = new InputSpec(ndim: 4); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { return keras.backend.spatial_2d_padding(inputs, padding: padding, diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropOutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropOutRNNCellMixin.cs new file mode 100644 index 00000000..b9a6fbc3 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropOutRNNCellMixin.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.Engine; + + + +namespace Tensorflow.Keras.Layers.Rnn +{ + public class DropoutRNNCellMixin + { + public float dropout; + public float recurrent_dropout; + // Get the dropout mask for RNN cell's input. + public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + { + if (dropout == 0f) + return null; + return _generate_dropout_mask( + tf.ones_like(input), + dropout, + training, + count); + } + + // Get the recurrent dropout mask for RNN cell. + public Tensors? get_recurrent_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + { + if (dropout == 0f) + return null; + return _generate_dropout_mask( + tf.ones_like(input), + recurrent_dropout, + training, + count); + } + + public Tensors _create_dropout_mask(Tensors input, bool training, int count = 1) + { + return _generate_dropout_mask( + tf.ones_like(input), + dropout, + training, + count); + } + + public Tensors _create_recurrent_dropout_mask(Tensors input, bool training, int count = 1) + { + return _generate_dropout_mask( + tf.ones_like(input), + recurrent_dropout, + training, + count); + } + + public Tensors _generate_dropout_mask(Tensor ones, float rate, bool training, int count = 1) + { + Tensors dropped_inputs() + { + DropoutArgs args = new DropoutArgs(); + args.Rate = rate; + var DropoutLayer = new Dropout(args); + var mask = DropoutLayer.Apply(ones, training: training); + return mask; + } + + if (count > 1) + { + Tensors results = new Tensors(); + for (int i = 0; i < count; i++) + { + results.Add(dropped_inputs()); + } + return results; + } + + return dropped_inputs(); + } + } + + +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 1f63932d..b9a6fbc3 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -1,38 +1,19 @@ using System; using System.Collections.Generic; -using System.Text; -using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; + + namespace Tensorflow.Keras.Layers.Rnn { - public abstract class DropoutRNNCellMixin: RnnCellBase + public class DropoutRNNCellMixin { public float dropout; public float recurrent_dropout; - // TODO(Rinne): deal with cache. - public DropoutRNNCellMixin(LayerArgs args): base(args) - { - - } - - protected void _create_non_trackable_mask_cache() - { - - } - - public void reset_dropout_mask() - { - - } - - public void reset_recurrent_dropout_mask() - { - - } - - public Tensors? get_dropout_mask_for_cell(Tensors input, bool training, int count = 1) + // Get the dropout mask for RNN cell's input. + public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) { if (dropout == 0f) return null; @@ -44,7 +25,7 @@ namespace Tensorflow.Keras.Layers.Rnn } // Get the recurrent dropout mask for RNN cell. - public Tensors? get_recurrent_dropout_mask_for_cell(Tensors input, bool training, int count = 1) + public Tensors? get_recurrent_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) { if (dropout == 0f) return null; @@ -97,4 +78,6 @@ namespace Tensorflow.Keras.Layers.Rnn return dropped_inputs(); } } + + } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 1449c908..773853b0 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -27,9 +27,15 @@ namespace Tensorflow.Keras.Layers.Rnn .ToArray(); } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return base.Call(inputs, initial_state: state, training: training); +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { + return base.Call(inputs, initial_state: initial_state, training: training); +>>>>>>> master } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 40b4d875..815297c1 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -1,16 +1,30 @@ +<<<<<<< HEAD using OneOf; using System; using System.Collections.Generic; using System.Reflection; using Tensorflow.Keras.ArgsDefinition; +======= +using System; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; +using static Tensorflow.Keras.ArgsDefinition.Rnn.RNNArgs; +>>>>>>> master using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Util; +<<<<<<< HEAD using Tensorflow.Common.Extensions; using System.Linq.Expressions; using Tensorflow.Keras.Utils; using Tensorflow.Common.Types; +======= +using OneOf; +using OneOf.Types; +using Tensorflow.Common.Extensions; +>>>>>>> master // from tensorflow.python.distribute import distribution_strategy_context as ds_context; namespace Tensorflow.Keras.Layers.Rnn @@ -22,6 +36,7 @@ namespace Tensorflow.Keras.Layers.Rnn /// public class RNN : RnnBase { +<<<<<<< HEAD private RNNArgs _args; private object _input_spec = null; // or NoneValue?? private object _state_spec = null; @@ -31,6 +46,17 @@ namespace Tensorflow.Keras.Layers.Rnn protected IVariableV1 _kernel; protected IVariableV1 _bias; protected IRnnCell _cell; +======= + private RNNArgs args; + private object input_spec = null; // or NoneValue?? + private object state_spec = null; + private Tensors _states = null; + private object constants_spec = null; + private int _num_constants = 0; + protected IVariableV1 kernel; + protected IVariableV1 bias; + protected ILayer cell; +>>>>>>> master public RNN(RNNArgs args) : base(PreConstruct(args)) { @@ -38,17 +64,51 @@ namespace Tensorflow.Keras.Layers.Rnn SupportsMasking = true; // if is StackedRnncell +<<<<<<< HEAD if (args.Cells != null) { _cell = new StackedRNNCells(new StackedRNNCellsArgs { Cells = args.Cells +======= + if (args.Cell.IsT0) + { + cell = new StackedRNNCells(new StackedRNNCellsArgs + { + Cells = args.Cell.AsT0, +>>>>>>> master }); } else { +<<<<<<< HEAD _cell = args.Cell; } +======= + cell = args.Cell.AsT1; + } + + Type type = cell.GetType(); + MethodInfo callMethodInfo = type.GetMethod("Call"); + if (callMethodInfo == null) + { + throw new ValueError(@"Argument `cell` or `cells`should have a `call` method. "); + } + + PropertyInfo state_size_info = type.GetProperty("state_size"); + if (state_size_info == null) + { + throw new ValueError(@"The RNN cell should have a `state_size` attribute"); + } + + + + // get input_shape + this.args = PreConstruct(args); + // The input shape is unknown yet, it could have nested tensor inputs, and + // the input spec will be the list of specs for nested inputs, the structure + // of the input_spec will be the same as the input. +>>>>>>> master // get input_shape _args = PreConstruct(args); @@ -169,9 +229,165 @@ namespace Tensorflow.Keras.Layers.Rnn } } + // States is a tuple consist of cell states_size, like (cell1.state_size, cell2.state_size,...) + // state_size can be a single integer, can also be a list/tuple of integers, can also be TensorShape or a list/tuple of TensorShape + public Tensors States + { + get + { + if (_states == null) + { + var state = nest.map_structure(x => null, cell.state_size); + return nest.is_nested(state) ? state : new Tensors { state }; + } + return _states; + } + set { _states = value; } + } + + private OneOf> compute_output_shape(Shape input_shape) + { + var batch = input_shape[0]; + var time_step = input_shape[1]; + if (args.TimeMajor) + { + (batch, time_step) = (time_step, batch); + } + + // state_size is a array of ints or a positive integer + var state_size = cell.state_size; + + // TODO(wanglongzhi2001),flat_output_size应该是什么类型的，Shape还是Tensor + Func _get_output_shape; + _get_output_shape = (flat_output_size) => + { + var output_dim = flat_output_size.as_int_list(); + Shape output_shape; + if (args.ReturnSequences) + { + if (args.TimeMajor) + { + output_shape = new Shape(new int[] { (int)time_step, (int)batch }.concat(output_dim)); + } + else + { + output_shape = new Shape(new int[] { (int)batch, (int)time_step }.concat(output_dim)); + + } + } + else + { + output_shape = new Shape(new int[] { (int)batch }.concat(output_dim)); + } + return output_shape; + }; + + Type type = cell.GetType(); + PropertyInfo output_size_info = type.GetProperty("output_size"); + Shape output_shape; + if (output_size_info != null) + { + output_shape = nest.map_structure(_get_output_shape, cell.output_size); + // TODO(wanglongzhi2001),output_shape应该简单的就是一个元组还是一个Shape类型 + output_shape = (output_shape.Length == 1 ? (int)output_shape[0] : output_shape); + } + else + { + output_shape = _get_output_shape(state_size[0]); + } + + if (args.ReturnState) + { + Func _get_state_shape; + _get_state_shape = (flat_state) => + { + var state_shape = new int[] { (int)batch }.concat(flat_state.as_int_list()); + return new Shape(state_shape); + }; + var state_shape = _get_state_shape(new Shape(state_size.ToArray())); + + return new List { output_shape, state_shape }; + } + else + { + return output_shape; + } + + } + + private Tensors compute_mask(Tensors inputs, Tensors mask) + { + // Time step masks must be the same for each input. + // This is because the mask for an RNN is of size [batch, time_steps, 1], + // and specifies which time steps should be skipped, and a time step + // must be skipped for all inputs. + + mask = nest.flatten(mask)[0]; + var output_mask = args.ReturnSequences ? mask : null; + if (args.ReturnState) + { + var state_mask = new List(); + for (int i = 0; i < len(States); i++) + { + state_mask.Add(null); + } + return new List { output_mask }.concat(state_mask); + } + else + { + return output_mask; + } + } + public override void build(KerasShapesWrapper input_shape) { object get_input_spec(Shape shape) +<<<<<<< HEAD +======= + { + var input_spec_shape = shape.as_int_list(); + + var (batch_index, time_step_index) = args.TimeMajor ? (1, 0) : (0, 1); + if (!args.Stateful) + { + input_spec_shape[batch_index] = -1; + } + input_spec_shape[time_step_index] = -1; + return new InputSpec(shape: input_spec_shape); + } + + Shape get_step_input_shape(Shape shape) + { + + // return shape[1:] if self.time_major else (shape[0],) + shape[2:] + if (args.TimeMajor) + { + return shape.as_int_list().ToList().GetRange(1, shape.Length - 1).ToArray(); + } + else + { + return new int[] { shape.as_int_list()[0] }.concat(shape.as_int_list().ToList().GetRange(2, shape.Length - 2).ToArray()); + } + + + } + + object get_state_spec(Shape shape) + { + var state_spec_shape = shape.as_int_list(); + // append bacth dim + state_spec_shape = new int[] { -1 }.concat(state_spec_shape); + return new InputSpec(shape: state_spec_shape); + + } + + // Check whether the input shape contains any nested shapes. It could be + // (tensor_shape(1, 2), tensor_shape(3, 4)) or (1, 2, 3) which is from + // numpy inputs. + + + if (!cell.Built) +>>>>>>> master { var input_spec_shape = shape.as_int_list(); @@ -220,6 +436,7 @@ namespace Tensorflow.Keras.Layers.Rnn } } +<<<<<<< HEAD /// /// /// @@ -243,16 +460,36 @@ namespace Tensorflow.Keras.Layers.Rnn //var (inputs_padded, row_length) = BackendImpl.convert_inputs_if_ragged(inputs); // 暂时先不接受ragged tensor int row_length = 0; // TODO(Rinne): support this param. +======= + // inputs: Tensors + // mask: Binary tensor of shape [batch_size, timesteps] indicating whether a given timestep should be masked + // training: bool + // initial_state: List of initial state tensors to be passed to the first call of the cell + // constants: List of constant tensors to be passed to the cell at each timestep + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { + //var (inputs_padded, row_length) = BackendImpl.convert_inputs_if_ragged(inputs); + // 暂时先不接受ragged tensor + int? row_length = null; +>>>>>>> master bool is_ragged_input = false; _validate_args_if_ragged(is_ragged_input, mask); (inputs, initial_state, constants) = _process_inputs(inputs, initial_state, constants); +<<<<<<< HEAD _maybe_reset_cell_dropout_mask(_cell); if (_cell is StackedRNNCells) { var stack_cell = _cell as StackedRNNCells; foreach (IRnnCell cell in stack_cell.Cells) +======= + _maybe_reset_cell_dropout_mask(cell); + if (cell is StackedRNNCells) + { + var stack_cell = cell as StackedRNNCells; + foreach (var cell in stack_cell.Cells) +>>>>>>> master { _maybe_reset_cell_dropout_mask(cell); } @@ -261,25 +498,43 @@ namespace Tensorflow.Keras.Layers.Rnn if (mask != null) { // Time step masks must be the same for each input. +<<<<<<< HEAD mask = mask.Flatten().First(); } Shape input_shape; if (!inputs.IsNested()) +======= + mask = nest.flatten(mask)[0]; + } + + Shape input_shape; + if (nest.is_nested(inputs)) +>>>>>>> master { // In the case of nested input, use the first element for shape check // input_shape = nest.flatten(inputs)[0].shape; // TODO(Wanglongzhi2001) +<<<<<<< HEAD input_shape = inputs.Flatten().First().shape; +======= + input_shape = nest.flatten(inputs)[0].shape; +>>>>>>> master } else { input_shape = inputs.shape; } +<<<<<<< HEAD var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; if (_args.Unroll && timesteps == null) +======= + var timesteps = args.TimeMajor ? input_shape[0] : input_shape[1]; + + if (args.Unroll && timesteps != null) +>>>>>>> master { throw new ValueError( "Cannot unroll a RNN if the " + @@ -297,6 +552,7 @@ namespace Tensorflow.Keras.Layers.Rnn } // cell_call_fn = (self.cell.__call__ if callable(self.cell) else self.cell.call) +<<<<<<< HEAD Func step; bool is_tf_rnn_cell = _cell.IsTFRnnCell; if (constants is not null) @@ -305,22 +561,51 @@ namespace Tensorflow.Keras.Layers.Rnn { throw new ValueError( $"RNN cell {_cell} does not support constants." + +======= + var cell_call_fn = cell.Call; + Func step; + if (constants != null) + { + ParameterInfo[] parameters = cell_call_fn.GetMethodInfo().GetParameters(); + bool hasParam = parameters.Any(p => p.Name == "constants"); + if (!hasParam) + { + throw new ValueError( + $"RNN cell {cell} does not support constants." + +>>>>>>> master $"Received: constants={constants}"); } step = (inputs, states) => { +<<<<<<< HEAD constants = new Tensors(states.TakeLast(_num_constants)); states = new Tensors(states.SkipLast(_num_constants)); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); return (output, new_states.Single); +======= + // constants = states[-self._num_constants :] + constants = states.numpy()[new Slice(states.Length - _num_constants, states.Length)]; + // states = states[: -self._num_constants] + states = states.numpy()[new Slice(0, states.Length - _num_constants)]; + // states = (states[0] if len(states) == 1 and is_tf_rnn_cell else states) + states = states.Length == 1 ? states[0] : states; + var (output, new_states) = cell_call_fn(inputs, null, null, states, constants); + // TODO(Wanglongzhi2001),should cell_call_fn's return value be Tensors, Tensors? + if (!nest.is_nested(new_states)) + { + return (output, new Tensors { new_states }); + } + return (output, new_states); +>>>>>>> master }; } else { step = (inputs, states) => { +<<<<<<< HEAD states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states.First()) : states; var (output, new_states) = _cell.Apply(inputs, states); return (output, new_states); @@ -350,14 +635,55 @@ namespace Tensorflow.Keras.Layers.Rnn { // TODO(Rinne): add go_backwards parameter and revise the `row_length` param output = keras.backend.maybe_convert_to_ragged(is_ragged_input, outputs, row_length, false); +======= + // states = (states[0] if len(states) == 1 and is_tf_rnn_cell else states) + states = states.Length == 1 ? states[0] : states; + var (output, new_states) = cell_call_fn(inputs, null, null, states, constants); + if (!nest.is_nested(new_states)) + { + return (output, new Tensors { new_states }); + } + return (output, new_states); + }; + } + + var (last_output, outputs, states) = BackendImpl.rnn(step, + inputs, + initial_state, + constants: constants, + go_backwards: args.GoBackwards, + mask: mask, + unroll: args.Unroll, + input_length: row_length != null ? new Tensor(row_length) : new Tensor(timesteps), + time_major: args.TimeMajor, + zero_output_for_mask: args.ZeroOutputForMask, + return_all_outputs: args.ReturnSequences); + + if (args.Stateful) + { + throw new NotImplementedException("this argument havn't been developed!"); + } + + Tensors output = new Tensors(); + if (args.ReturnSequences) + { + throw new NotImplementedException("this argument havn't been developed!"); + +>>>>>>> master } else { output = last_output; } +<<<<<<< HEAD if (_args.ReturnState) { +======= + if (args.ReturnState) + { + +>>>>>>> master foreach (var state in states) { output.Add(state); @@ -370,6 +696,7 @@ namespace Tensorflow.Keras.Layers.Rnn } } +<<<<<<< HEAD public override Tensors Apply(Tensors inputs, Tensors initial_states = null, bool training = false, IOptionalArgs? optional_args = null) { RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; @@ -401,25 +728,52 @@ namespace Tensorflow.Keras.Layers.Rnn { initial_state = new Tensors(inputs.Skip(1).SkipLast(_num_constants)); constants = new Tensors(inputs.TakeLast(_num_constants)); +======= + private (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensor inputs, Tensors initial_state, Tensors constants) + { + if (nest.is_sequence(input)) + { + if (_num_constants != 0) + { + initial_state = inputs[new Slice(1, len(inputs))]; + } + else + { + initial_state = inputs[new Slice(1, len(inputs) - _num_constants)]; + constants = inputs[new Slice(len(inputs) - _num_constants, len(inputs))]; +>>>>>>> master } if (len(initial_state) == 0) initial_state = null; inputs = inputs[0]; } +<<<<<<< HEAD if (_args.Stateful) +======= + + if (args.Stateful) +>>>>>>> master { if (initial_state != null) { var tmp = new Tensor[] { }; foreach (var s in nest.flatten(States)) { +<<<<<<< HEAD tmp.add(tf.math.count_nonzero(s.Single())); } var non_zero_count = tf.add_n(tmp); //initial_state = tf.cond(non_zero_count > 0, () => States, () => initial_state); if ((int)non_zero_count.numpy() > 0) +======= + tmp.add(tf.math.count_nonzero((Tensor)s)); + } + var non_zero_count = tf.add_n(tmp); + //initial_state = tf.cond(non_zero_count > 0, () => States, () => initial_state); + if((int)non_zero_count.numpy() > 0) +>>>>>>> master { initial_state = States; } @@ -428,6 +782,7 @@ namespace Tensorflow.Keras.Layers.Rnn { initial_state = States; } +<<<<<<< HEAD // TODO(Wanglongzhi2001), // initial_state = tf.nest.map_structure( //# When the layer has a inferred dtype, use the dtype from the @@ -440,15 +795,27 @@ namespace Tensorflow.Keras.Layers.Rnn } else if (initial_state is null) +======= + + } + else if(initial_state != null) +>>>>>>> master { initial_state = get_initial_state(inputs); } if (initial_state.Length != States.Length) { +<<<<<<< HEAD throw new ValueError($"Layer {this} expects {States.Length} state(s), " + $"but it received {initial_state.Length} " + $"initial state(s). Input received: {inputs}"); +======= + throw new ValueError( + $"Layer {this} expects {States.Length} state(s), " + + $"but it received {initial_state.Length} " + + $"initial state(s). Input received: {inputs}"); +>>>>>>> master } return (inputs, initial_state, constants); @@ -456,12 +823,20 @@ namespace Tensorflow.Keras.Layers.Rnn private void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) { +<<<<<<< HEAD if (!is_ragged_input) +======= + if (!is_ragged_input) +>>>>>>> master { return; } +<<<<<<< HEAD if (_args.Unroll) +======= + if (args.Unroll) +>>>>>>> master { throw new ValueError("The input received contains RaggedTensors and does " + "not support unrolling. Disable unrolling by passing " + @@ -479,11 +854,19 @@ namespace Tensorflow.Keras.Layers.Rnn void _maybe_reset_cell_dropout_mask(ILayer cell) { +<<<<<<< HEAD if (cell is DropoutRNNCellMixin CellDRCMixin) { CellDRCMixin.reset_dropout_mask(); CellDRCMixin.reset_recurrent_dropout_mask(); } +======= + //if (cell is DropoutRNNCellMixin) + //{ + // cell.reset_dropout_mask(); + // cell.reset_recurrent_dropout_mask(); + //} +>>>>>>> master } private static RNNArgs PreConstruct(RNNArgs args) @@ -516,6 +899,81 @@ namespace Tensorflow.Keras.Layers.Rnn public Tensors __call__(Tensors inputs, Tensor state = null, Tensor training = null) { throw new NotImplementedException(); +<<<<<<< HEAD +======= + } + + // 好像不能cell不能传接口类型 + //public RNN New(IRnnArgCell cell, + // bool return_sequences = false, + // bool return_state = false, + // bool go_backwards = false, + // bool stateful = false, + // bool unroll = false, + // bool time_major = false) + // => new RNN(new RNNArgs + // { + // Cell = cell, + // ReturnSequences = return_sequences, + // ReturnState = return_state, + // GoBackwards = go_backwards, + // Stateful = stateful, + // Unroll = unroll, + // TimeMajor = time_major + // }); + + //public RNN New(List cell, + // bool return_sequences = false, + // bool return_state = false, + // bool go_backwards = false, + // bool stateful = false, + // bool unroll = false, + // bool time_major = false) + // => new RNN(new RNNArgs + // { + // Cell = cell, + // ReturnSequences = return_sequences, + // ReturnState = return_state, + // GoBackwards = go_backwards, + // Stateful = stateful, + // Unroll = unroll, + // TimeMajor = time_major + // }); + + + protected Tensors get_initial_state(Tensor inputs) + { + Type type = cell.GetType(); + MethodInfo MethodInfo = type.GetMethod("get_initial_state"); + + if (nest.is_nested(inputs)) + { + // The input are nested sequences. Use the first element in the seq + // to get batch size and dtype. + inputs = nest.flatten(inputs)[0]; + } + + var input_shape = tf.shape(inputs); + var batch_size = args.TimeMajor ? input_shape[1] : input_shape[0]; + var dtype = inputs.dtype; + Tensor init_state; + if (MethodInfo != null) + { + init_state = (Tensor)MethodInfo.Invoke(cell, new object[] { null, batch_size, dtype }); + } + else + { + init_state = RNNUtils.generate_zero_filled_state(batch_size, cell.state_size, dtype); + } + + //if (!nest.is_nested(init_state)) + //{ + // init_state = new List { init_state}; + //} + return new List { init_state }; + + //return _generate_zero_filled_state_for_cell(null, null); +>>>>>>> master } // 好像不能cell不能传接口类型 @@ -585,7 +1043,11 @@ namespace Tensorflow.Keras.Layers.Rnn } // Check whether the state_size contains multiple states. +<<<<<<< HEAD public static bool is_multiple_state(GeneralizedTensorShape state_size) +======= + public static bool is_multiple_state(object state_size) +>>>>>>> master { return state_size.Shapes.Length > 1; } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNNUtils.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNNUtils.cs new file mode 100644 index 00000000..f516f765 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNNUtils.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Util; +using OneOf; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public class RNNUtils + { + public static Tensor generate_zero_filled_state(Tensor batch_size_tensor, StateSizeWrapper state_size, TF_DataType dtype = TF_DataType.TF_FLOAT) + { + if (batch_size_tensor == null || dtype == null) + { + throw new ValueError( + "batch_size and dtype cannot be None while constructing initial " + + $"state. Received: batch_size={batch_size_tensor}, dtype={dtype}"); + } + + Func create_zeros; + create_zeros = (StateSizeWrapper unnested_state_size) => + { + var flat_dims = unnested_state_size.state_size; + //if (unnested_state_size is int[]) + //{ + // flat_dims = new Shape(unnested_state_size.AsT0).as_int_list(); + //} + //else if (unnested_state_size.IsT1) + //{ + // flat_dims = new Shape(unnested_state_size.AsT1).as_int_list(); + //} + var init_state_size = batch_size_tensor.ToArray().concat(flat_dims); + return tf.zeros(init_state_size, dtype: dtype); + }; + + //if (nest.is_nested(state_size)) + //{ + // return nest.map_structure(create_zeros, state_size); + //} + //else + //{ + // return create_zeros(state_size); + //} + return create_zeros(state_size); + + } + + public static Tensor generate_zero_filled_state_for_cell(SimpleRNNCell cell, Tensors inputs, Tensor batch_size, TF_DataType dtype) + { + if (inputs != null) + { + batch_size = tf.shape(inputs)[0]; + dtype = inputs.dtype; + } + return generate_zero_filled_state(batch_size, cell.state_size, dtype); + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 193bff4f..cdf75756 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -4,9 +4,13 @@ using System.Text; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +<<<<<<< HEAD using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; using Tensorflow.Keras.Utils; +======= +using Tensorflow.Util; +>>>>>>> master namespace Tensorflow.Keras.Layers.Rnn { @@ -19,6 +23,7 @@ namespace Tensorflow.Keras.Layers.Rnn /// public class SimpleRNNCell : DropoutRNNCellMixin { +<<<<<<< HEAD SimpleRNNCellArgs _args; IVariableV1 _kernel; IVariableV1 _recurrent_kernel; @@ -34,15 +39,36 @@ namespace Tensorflow.Keras.Layers.Rnn public SimpleRNNCell(SimpleRNNCellArgs args) : base(args) { this._args = args; +======= + SimpleRNNArgs args; + IVariableV1 kernel; + IVariableV1 recurrent_kernel; + IVariableV1 bias; + DropoutRNNCellMixin DRCMixin; + public SimpleRNNCell(SimpleRNNArgs args) : base(args) + { + this.args = args; +>>>>>>> master if (args.Units <= 0) { throw new ValueError( $"units must be a positive integer, got {args.Units}"); } +<<<<<<< HEAD this._args.Dropout = Math.Min(1f, Math.Max(0f, this._args.Dropout)); this._args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this._args.RecurrentDropout)); _state_size = new GeneralizedTensorShape(args.Units); _output_size = new GeneralizedTensorShape(args.Units); +======= + this.args.Dropout = Math.Min(1f, Math.Max(0f, this.args.Dropout)); + this.args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this.args.RecurrentDropout)); + this.args.state_size = this.args.Units; + this.args.output_size = this.args.Units; + + DRCMixin = new DropoutRNNCellMixin(); + DRCMixin.dropout = this.args.Dropout; + DRCMixin.recurrent_dropout = this.args.RecurrentDropout; +>>>>>>> master } public override void build(KerasShapesWrapper input_shape) @@ -69,6 +95,7 @@ namespace Tensorflow.Keras.Layers.Rnn built = true; } +<<<<<<< HEAD // TODO(Rinne): revise the trining param (with refactoring of the framework) protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) { @@ -76,11 +103,20 @@ namespace Tensorflow.Keras.Layers.Rnn Tensors prev_output = Nest.IsNested(states) ? new Tensors(states[0]) : states; var dp_mask = get_dropout_mask_for_cell(inputs, training.Value); var rec_dp_mask = get_recurrent_dropout_mask_for_cell(prev_output, training.Value); +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) + { + Tensor states = initial_state[0]; + var prev_output = nest.is_nested(states) ? states[0] : states; + var dp_mask = DRCMixin.get_dropout_maskcell_for_cell(inputs, training.Value); + var rec_dp_mask = DRCMixin.get_recurrent_dropout_maskcell_for_cell(prev_output, training.Value); +>>>>>>> master Tensor h; var ranks = inputs.rank; if (dp_mask != null) { +<<<<<<< HEAD h = math_ops.matmul(math_ops.multiply(inputs.Single, dp_mask.Single), _kernel.AsTensor()); } @@ -92,10 +128,38 @@ namespace Tensorflow.Keras.Layers.Rnn if (_bias != null) { h = tf.nn.bias_add(h, _bias); +======= + if (ranks > 2) + { + // 因为multiply函数会自动添加第一个维度，所以加上下标0 + h = tf.linalg.tensordot(math_ops.multiply(inputs, dp_mask)[0], kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + h = math_ops.matmul(math_ops.multiply(inputs, dp_mask)[0], kernel.AsTensor()); + } + } + else + { + if (ranks > 2) + { + h = tf.linalg.tensordot(inputs, kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + h = math_ops.matmul(inputs, kernel.AsTensor()); + } + } + + if (bias != null) + { + h = tf.nn.bias_add(h, bias); +>>>>>>> master } if (rec_dp_mask != null) { +<<<<<<< HEAD prev_output = math_ops.multiply(prev_output, rec_dp_mask); } @@ -120,6 +184,38 @@ namespace Tensorflow.Keras.Layers.Rnn public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) { return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size.Value, dtype.Value); +======= + prev_output = math_ops.multiply(prev_output, rec_dp_mask)[0]; + } + + ranks = prev_output.rank; + Tensor output; + if (ranks > 2) + { + output = h + tf.linalg.tensordot(prev_output[0], recurrent_kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + output = h + math_ops.matmul(prev_output, recurrent_kernel.AsTensor()); + } + Console.WriteLine($"shape of output: {output.shape}"); + + if (args.Activation != null) + { + output = args.Activation.Apply(output); + } + if (nest.is_nested(states)) + { + return (output, new Tensors { output }); + } + return (output, output); + } + + + public Tensor get_initial_state(Tensors inputs, Tensor batch_size, TF_DataType dtype) + { + return RNNUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); +>>>>>>> master } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 56634853..61e47884 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -5,9 +5,10 @@ using System.Linq; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using static Tensorflow.Keras.ArgsDefinition.Rnn.RNNArgs; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +<<<<<<< HEAD using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Layers.Rnn @@ -15,6 +16,15 @@ namespace Tensorflow.Keras.Layers.Rnn public class StackedRNNCells : Layer, IRnnCell { public IList Cells { get; set; } +======= +using Tensorflow.Keras.ArgsDefinition.Rnn; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public class StackedRNNCells : Layer + { + public IList Cells { get; set; } +>>>>>>> master public bool reverse_state_order; public StackedRNNCells(StackedRNNCellsArgs args) : base(args) @@ -86,7 +96,11 @@ namespace Tensorflow.Keras.Layers.Rnn { return lastCell.OutputSize; } +<<<<<<< HEAD else if (RNN.is_multiple_state(lastCell.StateSize)) +======= + else if (RNN.is_multiple_state(lastCell.state_size)) +>>>>>>> master { return lastCell.StateSize.First(); //throw new NotImplementedException(""); @@ -98,7 +112,12 @@ namespace Tensorflow.Keras.Layers.Rnn } } +<<<<<<< HEAD public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) +======= + + public object get_initial_state() +>>>>>>> master { var cells = reverse_state_order ? Cells.Reverse() : Cells; Tensors initial_states = new Tensors(); @@ -118,7 +137,11 @@ namespace Tensorflow.Keras.Layers.Rnn return initial_states; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + public Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { // Recover per-cell states. var state_size = reverse_state_order ? StateSize.Reverse() : StateSize; diff --git a/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs b/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs index 6dfec319..67b57535 100644 --- a/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs +++ b/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs @@ -35,7 +35,11 @@ namespace Tensorflow.Keras.Layers built = true; } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { if (tf.Context.executing_eagerly()) return DeFunCall(inputs); diff --git a/src/TensorflowNET.Hub/KerasLayer.cs b/src/TensorflowNET.Hub/KerasLayer.cs index 20d9851b..5dcf6931 100644 --- a/src/TensorflowNET.Hub/KerasLayer.cs +++ b/src/TensorflowNET.Hub/KerasLayer.cs @@ -90,7 +90,11 @@ namespace Tensorflow.Hub } } +<<<<<<< HEAD protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optionalArgs = null) +======= + protected override Tensors Call(Tensors inputs, Tensor mask = null, bool? training = null, Tensors initial_state = null, Tensors constants = null) +>>>>>>> master { _check_trainability(); diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index f4980b82..03fdac3c 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -144,6 +144,34 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual(expected_output, actual_output); } +<<<<<<< HEAD +======= + [TestMethod] + public void SimpleRNNCell() + { + var cell = keras.layers.SimpleRNNCell(64, dropout:0.5f, recurrent_dropout:0.5f); + var h0 = new Tensors { tf.zeros(new Shape(4, 64)) }; + var x = tf.random.normal((4, 100)); + var (y, h1) = cell.Apply(inputs: x, state: h0); + // TODO(Wanglongzhi2001)，因为SimpleRNNCell需要返回一个Tensor和一个Tensors，只用一个Tensors的话 + // hold不住，所以自行在外面将h强制转换成Tensors + var h2 = (Tensors)h1; + Assert.AreEqual((4, 64), y.shape); + Assert.AreEqual((4, 64), h2[0].shape); + } + + [TestMethod, Ignore("WIP")] + public void SimpleRNN() + { + var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); + /*var simple_rnn = keras.layers.SimpleRNN(4); + var output = simple_rnn.Apply(inputs); + Assert.AreEqual((32, 4), output.shape);*/ + var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); + var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); + } + +>>>>>>> master [TestMethod] public void Resizing() { diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index d744c336..db7d5892 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -67,4 +67,8 @@ + + + +