Merge branch 'master' of https://github.com/SciSharp/TensorFlow.NET

src/TensorFlowNET.Core/APIs/tf.math.cs

@@ -348,6 +348,9 @@ namespace Tensorflow
         public static Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null) 
             => math_ops.cast(x, dtype, name);
 
+        public static Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null)
+            => math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name);
+
         public static Tensor argmax(Tensor input, int axis = -1, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
             => gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);
 

src/TensorFlowNET.Core/APIs/tf.nn.cs

@@ -75,7 +75,7 @@ namespace Tensorflow
             /// <param name="time_major"></param>
             /// <returns>A pair (outputs, state)</returns>
             public static (Tensor, Tensor) dynamic_rnn(RNNCell cell, Tensor inputs,
-                int? sequence_length = null, TF_DataType dtype = TF_DataType.DtInvalid,
+                Tensor sequence_length = null, TF_DataType dtype = TF_DataType.DtInvalid,
                 int? parallel_iterations = null, bool swap_memory = false, bool time_major = false)
                 => rnn.dynamic_rnn(cell, inputs, sequence_length: sequence_length, dtype: dtype,
                     parallel_iterations: parallel_iterations, swap_memory: swap_memory,

src/TensorFlowNET.Core/Operations/BasicRNNCell.cs

@@ -24,7 +24,8 @@ namespace Tensorflow
         int _num_units;
         Func<Tensor, string, Tensor> _activation;
 
-        protected override int state_size => _num_units;
+        public override int state_size => _num_units;
+        public override int output_size => _num_units;
 
         public BasicRNNCell(int num_units,
             Func<Tensor, string, Tensor> activation = null,

src/TensorFlowNET.Core/Operations/NnOps/rnn.cs

@@ -24,15 +24,15 @@ namespace Tensorflow.Operations
 {
     internal class rnn
     {
-        public static (Tensor, Tensor) dynamic_rnn(RNNCell cell, Tensor inputs,
-            int? sequence_length = null, Tensor initial_state = null, 
+        public static (Tensor, Tensor) dynamic_rnn(RNNCell cell, Tensor inputs_tensor,
+            Tensor sequence_length = null, Tensor initial_state = null, 
             TF_DataType dtype = TF_DataType.DtInvalid,
             int? parallel_iterations = null, bool swap_memory = false, bool time_major = false)
         {
             with(tf.variable_scope("rnn"), scope =>
             {
                 VariableScope varscope = scope;
-                var flat_input = nest.flatten(inputs);
+                var flat_input = nest.flatten(inputs_tensor);
 
                 if (!time_major)
                 {
@@ -42,24 +42,146 @@ namespace Tensorflow.Operations
 
                 parallel_iterations = parallel_iterations ?? 32;
 
-                if (sequence_length.HasValue)
+                if (sequence_length != null)
                     throw new NotImplementedException("dynamic_rnn sequence_length has value");
 
                 var batch_size = _best_effort_input_batch_size(flat_input);
 
+                Tensor state = null;
                 if (initial_state != null)
-                {
-                    var state = initial_state;
-                }
+                    state = initial_state;
                 else
+                    state = cell.get_initial_state(batch_size: batch_size, dtype: dtype);
+
+                var inputs = nest.pack_sequence_as(structure: inputs_tensor, flat_sequence: flat_input);
+
+                var (outputs, final_state) = _dynamic_rnn_loop(
+                    cell,
+                    inputs as Tensor,
+                    state,
+                    parallel_iterations: parallel_iterations.Value,
+                    swap_memory: swap_memory,
+                    sequence_length: sequence_length,
+                    dtype: dtype);
+            });
+
+            throw new NotImplementedException("");
+        }
+
+        /// <summary>
+        /// Internal implementation of Dynamic RNN.
+        /// </summary>
+        /// <param name="cell"></param>
+        /// <param name="inputs"></param>
+        /// <param name="initial_state"></param>
+        /// <param name="parallel_iterations"></param>
+        /// <param name="swap_memory"></param>
+        /// <param name="sequence_length"></param>
+        /// <param name="dtype"></param>
+        /// <returns></returns>
+        private static (Tensor, Tensor) _dynamic_rnn_loop(RNNCell cell, Tensor inputs, Tensor initial_state,
+            int parallel_iterations, bool swap_memory, Tensor sequence_length = null, TF_DataType dtype = TF_DataType.DtInvalid)
+        {
+            var state = initial_state;
+            var state_size = cell.state_size;
+
+            var flat_input = nest.flatten(inputs);
+            var flat_output_size = nest.flatten(cell.output_size);
+
+            // Construct an initial output
+            var input_shape = array_ops.shape(flat_input[0]);
+            var time_steps = input_shape.slice(0);
+            var batch_size = _best_effort_input_batch_size(flat_input);
+            var inputs_got_shape = flat_input.Select(input_ => input_.TensorShape.with_rank_at_least(3)).ToArray();
+
+            var dims = inputs_got_shape[0].Dimensions.Take(2).ToArray();
+            var (const_time_steps, const_batch_size) = (dims[0], dims[1]);
+
+            foreach(var shape in inputs_got_shape)
+            {
+                if (shape[2] == -1)
+                    throw new ValueError("Input size (depth of inputs) must be accessible via shape inference," +
+                        " but saw value None.");
+
+                var got_time_steps = shape.dims[0];
+                var got_batch_size = shape.dims[1];
+
+                if (const_time_steps != got_time_steps)
+                    throw new ValueError("Time steps is not the same for all the elements in the input in a " +
+                        "batch.");
+
+                if (const_batch_size != got_batch_size)
+                    throw new ValueError("Batch_size is not the same for all the elements in the input.");
+            }
+
+            Func<int, Tensor> _create_zero_arrays = (size_) =>
+            {
+                var size = rnn_cell_impl._concat(batch_size, size_);
+                return array_ops.zeros(
+                    array_ops.stack(size), dtype: _infer_state_dtype(dtype, state));
+            };
+
+            // Prepare dynamic conditional copying of state & output
+            var flat_zero_output = flat_output_size.Select(output => _create_zero_arrays(output)).ToArray();
+            var zero_output = nest.pack_sequence_as(structure: cell.output_size, flat_sequence: flat_zero_output);
+
+            Tensor min_sequence_length = null, max_sequence_length = null;
+            if (sequence_length != null)
+            {
+                min_sequence_length = math_ops.reduce_min(sequence_length);
+                max_sequence_length = math_ops.reduce_max(sequence_length);
+            }
+            else
+            {
+                max_sequence_length = time_steps;
+            }
+
+            var time = array_ops.constant(0, dtype: dtypes.int32, name: "time");
+
+            string base_name = null;
+            with(ops.name_scope("dynamic_rnn"), scope => base_name = scope);
+
+            Func<string, TensorShape, TF_DataType, Tensor> _create_ta = (name, element_shape, dtype_) =>
+            {
+                new TensorArray(dtype: dtype_,
+                                        size: time_steps,
+                                        element_shape: element_shape,
+                                        tensor_array_name: base_name + name);
+                throw new NotImplementedException("");
+            };
+
+            bool in_graph_mode = true;
+            if (in_graph_mode)
+            {
+                foreach(var (i, out_size) in enumerate(flat_output_size))
                 {
-                    cell.get_initial_state(batch_size: batch_size, dtype: dtype);
+                    _create_ta($"output_{i}",
+                        new TensorShape(const_batch_size).concatenate(
+                            _maybe_tensor_shape_from_tensor(out_size)),
+                        _infer_state_dtype(dtype, state));
+
+
+                    
                 }
-            });
+            }
 
             throw new NotImplementedException("");
         }
 
+        private static TensorShape _maybe_tensor_shape_from_tensor(Tensor shape)
+            => shape.TensorShape;
+
+        private static TensorShape _maybe_tensor_shape_from_tensor(int shape)
+            => new TensorShape(shape);
+
+        private static TF_DataType _infer_state_dtype(TF_DataType explicit_dtype, Tensor state)
+        {
+            if (explicit_dtype != TF_DataType.DtInvalid)
+                return explicit_dtype;
+
+            throw new NotImplementedException("_infer_state_dtype");
+        }
+
         /// <summary>
         /// Transposes the batch and time dimensions of a Tensor.
         /// </summary>

src/TensorFlowNET.Core/Operations/NnOps/rnn_cell_impl.cs

@@ -0,0 +1,57 @@
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+
+namespace Tensorflow.Operations
+{
+    public class rnn_cell_impl
+    {
+        public BasicRNNCell BasicRNNCell(int num_units)
+            => new BasicRNNCell(num_units);
+
+        public static Tensor _concat(Tensor prefix, int suffix, bool @static = false)
+        {
+            var p = prefix;
+            var p_static = tensor_util.constant_value(prefix);
+            if (p.NDims == 0)
+                p = array_ops.expand_dims(p, 0);
+            else if (p.NDims != 1)
+                throw new ValueError($"prefix tensor must be either a scalar or vector, but saw tensor: {p}");
+
+            var s_tensor_shape = new TensorShape(suffix);
+            var s_static = s_tensor_shape.NDim > -1 ?
+                s_tensor_shape.Dimensions :
+                null;
+            var s = s_tensor_shape.is_fully_defined() ?
+                constant_op.constant(s_tensor_shape.Dimensions, dtype: dtypes.int32) :
+                null;
+
+            if (@static)
+            {
+                if (p_static is null) return null;
+                var shape = new TensorShape(p_static).concatenate(s_static);
+                throw new NotImplementedException("RNNCell _concat");
+            }
+            else
+            {
+                if (p is null || s is null)
+                    throw new ValueError($"Provided a prefix or suffix of None: {prefix} and {suffix}");
+                return array_ops.concat(new[] { p, s }, 0);
+            }
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/RNNCell.cs

@@ -15,6 +15,7 @@
 ******************************************************************************/
 
 using System;
+using Tensorflow.Operations;
 using Tensorflow.Util;
 using static Tensorflow.Python;
 
@@ -48,7 +49,9 @@ namespace Tensorflow
         /// difference between TF and Keras RNN cell.
         /// </summary>
         protected bool _is_tf_rnn_cell = false;
-        protected virtual int state_size { get; }
+        public virtual int state_size { get; }
+
+        public virtual int output_size { get; }
 
         public RNNCell(bool trainable = true,
             string name = null,
@@ -89,12 +92,18 @@ namespace Tensorflow
 
         private Tensor _zero_state_tensors(int state_size, Tensor batch_size, TF_DataType dtype)
         {
-            nest.map_structure(x =>
+            var output = nest.map_structure(s =>
             {
-                throw new NotImplementedException("");
+                var c = rnn_cell_impl._concat(batch_size, s);
+                var size = array_ops.zeros(c, dtype: dtype);
+
+                var c_static = rnn_cell_impl._concat(batch_size, s, @static: true);
+                size.set_shape(c_static);
+
+                return size;
             }, state_size);
 
-            throw new NotImplementedException("");
+            return output;
         }
     }
 }

src/TensorFlowNET.Core/Operations/TensorArray.cs

@@ -0,0 +1,54 @@
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Operations
+{
+    /// <summary>
+    /// TensorArray is designed to hide an underlying implementation object
+    /// and as such accesses many of that object's hidden fields.
+    ///
+    /// "Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays.
+    /// This class is meant to be used with dynamic iteration primitives such as
+    /// `while_loop` and `map_fn`.  It supports gradient back-propagation via special
+    /// "flow" control flow dependencies.
+    /// </summary>
+    public class TensorArray
+    {
+        _GraphTensorArray _implementation;
+
+        public TensorArray(TF_DataType dtype, Tensor size = null, bool? clear_after_read = null, bool? dynamic_size = null,
+            string tensor_array_name = null, Tensor handle = null, Tensor flow = null,
+            bool infer_shape = true, TensorShape element_shape = null,
+            bool colocate_with_first_write_call = true, string name = null)
+        {
+            _implementation = new _GraphTensorArray(dtype, 
+                size: size,
+                dynamic_size: dynamic_size,
+                clear_after_read: clear_after_read,
+                tensor_array_name: tensor_array_name,
+                handle: handle,
+                flow: flow,
+                infer_shape: infer_shape,
+                element_shape: element_shape,
+                colocate_with_first_write_call: colocate_with_first_write_call,
+                name: name);
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs

@@ -0,0 +1,102 @@
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Python;
+
+namespace Tensorflow.Operations
+{
+    internal class _GraphTensorArray
+    {
+        TF_DataType _dtype;
+
+        /// <summary>
+        /// Used to keep track of what tensors the TensorArray should be
+        /// colocated with.  We choose to colocate the TensorArray with the
+        /// first tensor written to it.
+        /// </summary>
+        bool _colocate_with_first_write_call;
+
+        bool _infer_shape;
+        List<TensorShape> _element_shape;
+
+        object _colocate_with;
+
+        public _GraphTensorArray(TF_DataType dtype, Tensor size, bool? dynamic_size = null,
+            bool? clear_after_read = null, string tensor_array_name = null, Tensor handle = null, Tensor flow = null, 
+            bool infer_shape = true, TensorShape element_shape = null, 
+            bool colocate_with_first_write_call = true, string name = null)
+        {
+            clear_after_read = clear_after_read ?? true;
+            dynamic_size = dynamic_size ?? false;
+
+            _dtype = dtype;
+
+            _colocate_with_first_write_call = colocate_with_first_write_call;
+            if (colocate_with_first_write_call)
+                _colocate_with = new Tensor[0];
+
+            // Record the current static shape for the array elements. The element
+            // shape is defined either by `element_shape` or the shape of the tensor
+            // of the first write. If `infer_shape` is true, all writes checks for
+            // shape equality.
+            if(element_shape == null)
+            {
+                _infer_shape = infer_shape;
+                _element_shape = new List<TensorShape> { };
+            }
+            else
+            {
+                _infer_shape = true;
+                _element_shape = new List<TensorShape> { };
+            }
+
+            with(ops.name_scope(name, "", new { handle, size, flow }), scope =>
+            {
+                if(handle != null)
+                {
+
+                }
+                else
+                {
+                    Func<(Tensor, Tensor)> create = () => gen_data_flow_ops.tensor_array_v3(size,
+                        dtype: dtype,
+                        element_shape: element_shape,
+                        identical_element_shapes: infer_shape,
+                        dynamic_size: dynamic_size.Value,
+                        clear_after_read: clear_after_read.Value,
+                        tensor_array_name: tensor_array_name,
+                        name: scope);
+
+                    // Construct the TensorArray with an empty device.  The first
+                    // write into the TensorArray from a Tensor with a set device
+                    // will retroactively set the device value of this op.
+                    if (colocate_with_first_write_call)
+                    {
+                        ops.colocate_with(ignore_existing: true);
+                        create();
+                    }
+                    else
+                    {
+
+                    }
+                }
+            });
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/array_ops.py.cs

@@ -29,6 +29,17 @@ namespace Tensorflow
         public static Tensor prevent_gradient(Tensor input, string message = "", string name = null)
             => gen_array_ops.prevent_gradient(input, message: message, name: name);
 
+        internal static Tensor constant(object value,
+            TF_DataType dtype = TF_DataType.DtInvalid,
+            int[] shape = null,
+            string name = "Const",
+            bool verify_shape = false) => constant_op._constant_impl(value,
+                dtype,
+                shape,
+                name,
+                verify_shape: verify_shape,
+                allow_broadcast: false);
+
         public static Tensor zeros(Shape shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null)
         {
             dtype = dtype.as_base_dtype();

src/TensorFlowNET.Core/Operations/gen_data_flow_ops.py.cs → src/TensorFlowNET.Core/Operations/gen_data_flow_ops.cs

@@ -27,5 +27,23 @@ namespace Tensorflow
 
             return _op.outputs[0];
         }
+
+        public static (Tensor, Tensor) tensor_array_v3(Tensor size, TF_DataType dtype = TF_DataType.DtInvalid, 
+            int[] element_shape = null, bool dynamic_size = false, bool clear_after_read = true, 
+            bool identical_element_shapes = false, string tensor_array_name = "tensor_array_name", string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("TensorArrayV3", name, new
+            {
+                size,
+                dtype,
+                element_shape,
+                dynamic_size,
+                clear_after_read,
+                identical_element_shapes,
+                tensor_array_name
+            });
+
+            return (null, null);
+        }
     }
 }

src/TensorFlowNET.Core/Operations/gen_math_ops.cs

@@ -238,6 +238,13 @@ namespace Tensorflow
             return _op.outputs[0];
         }
 
+        public static Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("Cumsum", name, args: new { x, axis, exclusive, reverse });
+
+            return _op.outputs[0];
+        }
+
         /// <summary>
         /// Computes the sum along segments of a tensor.
         /// </summary>

src/TensorFlowNET.Core/Operations/math_ops.cs

@@ -80,6 +80,17 @@ namespace Tensorflow
             });
         }
 
+        public static Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null)
+        {
+            return with(ops.name_scope(name, "Cumsum", new {x}), scope =>
+            {
+                name = scope;
+                x = ops.convert_to_tensor(x, name: "x");
+
+                return gen_math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name);
+            });
+        }
+
         /// <summary>
         /// Computes Psi, the derivative of Lgamma (the log of the absolute value of
         /// `Gamma(x)`), element-wise.

src/TensorFlowNET.Core/Operations/rnn_cell_impl.cs

@@ -1,8 +0,0 @@
-namespace Tensorflow.Operations
-{
-    public class rnn_cell_impl
-    {
-        public BasicRNNCell BasicRNNCell(int num_units)
-            => new BasicRNNCell(num_units);
-    }
-}

src/TensorFlowNET.Core/Sessions/BaseSession.cs

@@ -1,329 +1,329 @@
-/*****************************************************************************
-   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-******************************************************************************/
-
-using NumSharp;
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Numerics;
-using System.Text;
-
-namespace Tensorflow
-{
-    public class BaseSession
-    {
-        protected Graph _graph;
-        protected bool _opened;
-        protected bool _closed;
-        protected int _current_version;
-        protected byte[] _target;
-        protected IntPtr _session;
-        public Status Status;
-        public Graph graph => _graph;
-
-        public BaseSession(string target = "", Graph g = null, SessionOptions opts = null)
-        {
-            _graph = g is null ? ops.get_default_graph() : g;
-
-            _target = UTF8Encoding.UTF8.GetBytes(target);
-
-            SessionOptions newOpts = null;
-            if (opts == null)
-                newOpts = c_api.TF_NewSessionOptions();
-
-            Status = new Status();
-
-            _session = c_api.TF_NewSession(_graph, opts ?? newOpts, Status);
-
-            // dispose newOpts
-            if (opts == null)
-                c_api.TF_DeleteSessionOptions(newOpts);
-
-            Status.Check(true);
-        }
-
-        public virtual NDArray run(object fetches, params FeedItem[] feed_dict)
-        {
-            return _run(fetches, feed_dict);
-        }
-
-        public virtual NDArray run(object fetches, Hashtable feed_dict = null)
-        {
-            var feed_items = feed_dict == null ? new FeedItem[0] :
-                feed_dict.Keys.OfType<object>().Select(key => new FeedItem(key, feed_dict[key])).ToArray();
-            return _run(fetches, feed_items);
-        }
-
-        private NDArray _run(object fetches, FeedItem[] feed_dict = null)
-        {
-            var feed_dict_tensor = new Dictionary<object, object>();
-            var feed_map = new Dictionary<object, object>();
-
-            Func<FeedItem, IEnumerable<(object, object)>> feed_fn = (item) =>
-            {
-                return new (object, object)[] { (item.Key, item.Value) };
-            };
-
-            // Validate and process feed_dict.
-            if (feed_dict != null)
-            {
-                foreach (var feed in feed_dict)
-                {
-                    foreach (var (subfeed, subfeed_val) in feed_fn(feed))
-                    {
-                        var subfeed_t = _graph.as_graph_element(subfeed, allow_tensor: true, allow_operation: false);
-                        //var subfeed_dtype = subfeed_t.dtype.as_numpy_datatype(); // subfeed_dtype was never used
-                        feed_dict_tensor[subfeed_t] = subfeed_val;
-                        feed_map[subfeed_t.name] = (subfeed_t, subfeed_val);
-                    }
-                }
-            }
-
-            // Create a fetch handler to take care of the structure of fetches.
-            var fetch_handler = new _FetchHandler(_graph, fetches, feed_dict_tensor);
-
-            // Run request and get response.
-            // We need to keep the returned movers alive for the following _do_run().
-            // These movers are no longer needed when _do_run() completes, and
-            // are deleted when `movers` goes out of scope when this _run() ends.
-            var _ = _update_with_movers();
-            var final_fetches = fetch_handler.fetches();
-            var final_targets = fetch_handler.targets();
-
-            // We only want to really perform the run if fetches or targets are provided,
-            // or if the call is a partial run that specifies feeds.
-            var results = _do_run(final_targets.Select(x => (Operation)x).ToList(), final_fetches, feed_dict_tensor);
-
-            return fetch_handler.build_results(this, results);
-        }
-
-        /// <summary>
-        /// Runs a step based on the given fetches and feeds.
-        /// </summary>
-        /// <typeparam name="T"></typeparam>
-        /// <param name="target_list">A list of operations to be run, but not fetched.</param>
-        /// <param name="fetch_list"></param>
-        /// <param name="feed_dict"></param>
-        /// <returns>
-        /// A list of numpy ndarrays, corresponding to the elements of
-        /// `fetch_list`.  If the ith element of `fetch_list` contains the
-        /// name of an operation, the first Tensor output of that operation
-        /// will be returned for that element.
-        /// </returns>
-        private NDArray[] _do_run(List<Operation> target_list, List<Tensor> fetch_list, Dictionary<object, object> feed_dict)
-        {
-            var feeds = feed_dict.Select(x =>
-            {
-                if (x.Key is Tensor tensor)
-                {
-                    switch (x.Value)
-                    {
-#if _REGEN
-        %types=["sbyte", "byte", "short", "ushort", "int", "uint", "long", "ulong", "float", "double", "Complex"]
-        %foreach types%
-                        case #1 v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case #1[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-        %
-#else
-                        case sbyte v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case sbyte[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case byte v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case byte[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case short v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case short[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case ushort v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case ushort[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case int v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case int[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case uint v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case uint[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case long v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case long[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case ulong v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case ulong[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case float v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case float[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case double v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case double[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case Complex v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case Complex[] v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-#endif
-                        case bool v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor((byte)(v?1:0), TF_DataType.TF_BOOL));
-                        case string v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case IntPtr v:
-                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
-                        case Tensor v:
+/*****************************************************************************
+   Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+******************************************************************************/
+
+using NumSharp;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+
+namespace Tensorflow
+{
+    public class BaseSession
+    {
+        protected Graph _graph;
+        protected bool _opened;
+        protected bool _closed;
+        protected int _current_version;
+        protected byte[] _target;
+        protected IntPtr _session;
+        public Status Status;
+        public Graph graph => _graph;
+
+        public BaseSession(string target = "", Graph g = null, SessionOptions opts = null)
+        {
+            _graph = g is null ? ops.get_default_graph() : g;
+
+            _target = UTF8Encoding.UTF8.GetBytes(target);
+
+            SessionOptions newOpts = null;
+            if (opts == null)
+                newOpts = c_api.TF_NewSessionOptions();
+
+            Status = new Status();
+
+            _session = c_api.TF_NewSession(_graph, opts ?? newOpts, Status);
+
+            // dispose newOpts
+            if (opts == null)
+                c_api.TF_DeleteSessionOptions(newOpts);
+
+            Status.Check(true);
+        }
+
+        public virtual NDArray run(object fetches, params FeedItem[] feed_dict)
+        {
+            return _run(fetches, feed_dict);
+        }
+
+        public virtual NDArray run(object fetches, Hashtable feed_dict = null)
+        {
+            var feed_items = feed_dict == null ? new FeedItem[0] :
+                feed_dict.Keys.OfType<object>().Select(key => new FeedItem(key, feed_dict[key])).ToArray();
+            return _run(fetches, feed_items);
+        }
+
+        private NDArray _run(object fetches, FeedItem[] feed_dict = null)
+        {
+            var feed_dict_tensor = new Dictionary<object, object>();
+            var feed_map = new Dictionary<object, object>();
+
+            Func<FeedItem, IEnumerable<(object, object)>> feed_fn = (item) =>
+            {
+                return new (object, object)[] { (item.Key, item.Value) };
+            };
+
+            // Validate and process feed_dict.
+            if (feed_dict != null)
+            {
+                foreach (var feed in feed_dict)
+                {
+                    foreach (var (subfeed, subfeed_val) in feed_fn(feed))
+                    {
+                        var subfeed_t = _graph.as_graph_element(subfeed, allow_tensor: true, allow_operation: false);
+                        //var subfeed_dtype = subfeed_t.dtype.as_numpy_datatype(); // subfeed_dtype was never used
+                        feed_dict_tensor[subfeed_t] = subfeed_val;
+                        feed_map[subfeed_t.name] = (subfeed_t, subfeed_val);
+                    }
+                }
+            }
+
+            // Create a fetch handler to take care of the structure of fetches.
+            var fetch_handler = new _FetchHandler(_graph, fetches, feed_dict_tensor);
+
+            // Run request and get response.
+            // We need to keep the returned movers alive for the following _do_run().
+            // These movers are no longer needed when _do_run() completes, and
+            // are deleted when `movers` goes out of scope when this _run() ends.
+            var _ = _update_with_movers();
+            var final_fetches = fetch_handler.fetches();
+            var final_targets = fetch_handler.targets();
+
+            // We only want to really perform the run if fetches or targets are provided,
+            // or if the call is a partial run that specifies feeds.
+            var results = _do_run(final_targets.Select(x => (Operation)x).ToList(), final_fetches, feed_dict_tensor);
+
+            return fetch_handler.build_results(this, results);
+        }
+
+        /// <summary>
+        /// Runs a step based on the given fetches and feeds.
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="target_list">A list of operations to be run, but not fetched.</param>
+        /// <param name="fetch_list"></param>
+        /// <param name="feed_dict"></param>
+        /// <returns>
+        /// A list of numpy ndarrays, corresponding to the elements of
+        /// `fetch_list`.  If the ith element of `fetch_list` contains the
+        /// name of an operation, the first Tensor output of that operation
+        /// will be returned for that element.
+        /// </returns>
+        private NDArray[] _do_run(List<Operation> target_list, List<Tensor> fetch_list, Dictionary<object, object> feed_dict)
+        {
+            var feeds = feed_dict.Select(x =>
+            {
+                if (x.Key is Tensor tensor)
+                {
+                    switch (x.Value)
+                    {
+#if _REGEN
+        %types=["sbyte", "byte", "short", "ushort", "int", "uint", "long", "ulong", "float", "double", "Complex"]
+        %foreach types%
+                        case #1 v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case #1[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+        %
+#else
+                        case sbyte v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case sbyte[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case byte v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case byte[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case short v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case short[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case ushort v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case ushort[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case int v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case int[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case uint v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case uint[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case long v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case long[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case ulong v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case ulong[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case float v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case float[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case double v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case double[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case Complex v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case Complex[] v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+#endif
+                        case bool v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor((byte)(v?1:0), TF_DataType.TF_BOOL));
+                        case string v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case IntPtr v:
+                            return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v));
+                        case Tensor v:
                             return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), v);
-                        case NDArray v:
-                                return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v, tensor.dtype));
-                        default:
-                            throw new NotImplementedException($"feed_dict data type {(x.Value?.GetType().Name ?? "<null>")}");
-                    }
-                }
-                throw new NotImplementedException("_do_run.feed_dict");
-            }).ToArray();
-            var fetches = fetch_list.Select(x => x._as_tf_output()).ToArray();
-            var targets = target_list;
-
-            return _call_tf_sessionrun(feeds, fetches, target_list);
-        }
-
-        private unsafe NDArray[] _call_tf_sessionrun(KeyValuePair<TF_Output, Tensor>[] feed_dict, TF_Output[] fetch_list, List<Operation> target_list)
-        {
-            // Ensure any changes to the graph are reflected in the runtime.
-            _extend_graph();
-
-            var status = new Status();
-
-            var output_values = fetch_list.Select(x => IntPtr.Zero).ToArray();
-
-            c_api.TF_SessionRun(_session,
-                run_options: null,
-                inputs: feed_dict.Select(f => f.Key).ToArray(),
-                input_values: feed_dict.Select(f => (IntPtr)f.Value).ToArray(),
-                ninputs: feed_dict.Length,
-                outputs: fetch_list,
-                output_values: output_values,
-                noutputs: fetch_list.Length,
-                target_opers: target_list.Select(f => (IntPtr)f).ToArray(),
-                ntargets: target_list.Count,
-                run_metadata: IntPtr.Zero,
-                status: status);
-
-            status.Check(true);
-
-            var result = new NDArray[fetch_list.Length];
-
-            for (int i = 0; i < fetch_list.Length; i++)
-                result[i] = fetchValue(output_values[i]);
-
-            for (int i = 0; i < feed_dict.Length; i++)
-                feed_dict[i].Value.Dispose();
-
-            return result;
-        }
-
-        private unsafe NDArray fetchValue(IntPtr output)
-        {
-            var tensor = new Tensor(output);
-            NDArray nd = null;
-            Type type = tensor.dtype.as_numpy_datatype();
-            var ndims = tensor.shape.Select(x => (int)x).ToArray();
-            var offset = c_api.TF_TensorData(output);
-
-            switch (tensor.dtype)
-            {
-                case TF_DataType.TF_BOOL:
-                    var bools = new bool[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        bools[i] = *(bool*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(bools).reshape(ndims);
-                    break;
-                case TF_DataType.TF_STRING:
-                    var bytes = tensor.Data();
-                    // wired, don't know why we have to start from offset 9.
-                    // length in the begin
-                    var str = UTF8Encoding.Default.GetString(bytes, 9, bytes[8]);
-                    nd = np.array(str).reshape();
-                    break;
-                case TF_DataType.TF_UINT8:
-                    var _bytes = new byte[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        _bytes[i] = *(byte*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(_bytes).reshape(ndims);
-                    break;
-                case TF_DataType.TF_INT16:
-                    var shorts = new short[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        shorts[i] = *(short*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(shorts).reshape(ndims);
-                    break;
-                case TF_DataType.TF_INT32:
-                    var ints = new int[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        ints[i] = *(int*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(ints).reshape(ndims);
-                    break;
-                case TF_DataType.TF_INT64:
-                    var longs = new long[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        longs[i] = *(long*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(longs).reshape(ndims);
-                    break;
-                case TF_DataType.TF_FLOAT:
-                    var floats = new float[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        floats[i] = *(float*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(floats).reshape(ndims);
-                    break;
-                case TF_DataType.TF_DOUBLE:
-                    var doubles = new double[tensor.size];
-                    for (ulong i = 0; i < tensor.size; i++)
-                        doubles[i] = *(double*)(offset + (int)(tensor.itemsize * i));
-                    nd = np.array(doubles).reshape(ndims);
-                    break;
-                default:
-                    throw new NotImplementedException("can't fetch output");
-            }
-
-            tensor.Dispose();
-
-            return nd;
-        }
-
-        /// <summary>
-        /// If a tensor handle that is fed to a device incompatible placeholder, 
-        /// we move the tensor to the right device, generate a new tensor handle, 
-        /// and update feed_dict to use the new handle.
-        /// </summary>
-        private List<object> _update_with_movers()
-        {
-            return new List<object> { };
-        }
-
-        private void _extend_graph()
-        {
-
-        }
-    }
-}
+                        case NDArray v:
+                                return new KeyValuePair<TF_Output, Tensor>(tensor._as_tf_output(), new Tensor(v, tensor.dtype));
+                        default:
+                            throw new NotImplementedException($"feed_dict data type {(x.Value?.GetType().Name ?? "<null>")}");
+                    }
+                }
+                throw new NotImplementedException("_do_run.feed_dict");
+            }).ToArray();
+            var fetches = fetch_list.Select(x => x._as_tf_output()).ToArray();
+            var targets = target_list;
+
+            return _call_tf_sessionrun(feeds, fetches, target_list);
+        }
+
+        private unsafe NDArray[] _call_tf_sessionrun(KeyValuePair<TF_Output, Tensor>[] feed_dict, TF_Output[] fetch_list, List<Operation> target_list)
+        {
+            // Ensure any changes to the graph are reflected in the runtime.
+            _extend_graph();
+
+            var status = new Status();
+
+            var output_values = fetch_list.Select(x => IntPtr.Zero).ToArray();
+
+            c_api.TF_SessionRun(_session,
+                run_options: null,
+                inputs: feed_dict.Select(f => f.Key).ToArray(),
+                input_values: feed_dict.Select(f => (IntPtr)f.Value).ToArray(),
+                ninputs: feed_dict.Length,
+                outputs: fetch_list,
+                output_values: output_values,
+                noutputs: fetch_list.Length,
+                target_opers: target_list.Select(f => (IntPtr)f).ToArray(),
+                ntargets: target_list.Count,
+                run_metadata: IntPtr.Zero,
+                status: status);
+
+            status.Check(true);
+
+            var result = new NDArray[fetch_list.Length];
+
+            for (int i = 0; i < fetch_list.Length; i++)
+                result[i] = fetchValue(output_values[i]);
+
+            for (int i = 0; i < feed_dict.Length; i++)
+                feed_dict[i].Value.Dispose();
+
+            return result;
+        }
+
+        private unsafe NDArray fetchValue(IntPtr output)
+        {
+            var tensor = new Tensor(output);
+            NDArray nd = null;
+            Type type = tensor.dtype.as_numpy_datatype();
+            var ndims = tensor.shape.Select(x => (int)x).ToArray();
+            var offset = c_api.TF_TensorData(output);
+
+            switch (tensor.dtype)
+            {
+                case TF_DataType.TF_BOOL:
+                    var bools = new bool[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        bools[i] = *(bool*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(bools).reshape(ndims);
+                    break;
+                case TF_DataType.TF_STRING:
+                    var bytes = tensor.Data();
+                    // wired, don't know why we have to start from offset 9.
+                    // length in the begin
+                    var str = UTF8Encoding.Default.GetString(bytes, 9, bytes[8]);
+                    nd = np.array(str).reshape();
+                    break;
+                case TF_DataType.TF_UINT8:
+                    var _bytes = new byte[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        _bytes[i] = *(byte*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(_bytes).reshape(ndims);
+                    break;
+                case TF_DataType.TF_INT16:
+                    var shorts = new short[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        shorts[i] = *(short*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(shorts).reshape(ndims);
+                    break;
+                case TF_DataType.TF_INT32:
+                    var ints = new int[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        ints[i] = *(int*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(ints).reshape(ndims);
+                    break;
+                case TF_DataType.TF_INT64:
+                    var longs = new long[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        longs[i] = *(long*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(longs).reshape(ndims);
+                    break;
+                case TF_DataType.TF_FLOAT:
+                    var floats = new float[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        floats[i] = *(float*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(floats).reshape(ndims);
+                    break;
+                case TF_DataType.TF_DOUBLE:
+                    var doubles = new double[tensor.size];
+                    for (ulong i = 0; i < tensor.size; i++)
+                        doubles[i] = *(double*)(offset + (int)(tensor.itemsize * i));
+                    nd = np.array(doubles).reshape(ndims);
+                    break;
+                default:
+                    throw new NotImplementedException("can't fetch output");
+            }
+
+            tensor.Dispose();
+
+            return nd;
+        }
+
+        /// <summary>
+        /// If a tensor handle that is fed to a device incompatible placeholder, 
+        /// we move the tensor to the right device, generate a new tensor handle, 
+        /// and update feed_dict to use the new handle.
+        /// </summary>
+        private List<object> _update_with_movers()
+        {
+            return new List<object> { };
+        }
+
+        private void _extend_graph()
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Tensors/Tensor.cs

@@ -110,6 +110,11 @@ namespace Tensorflow
             this.shape = shape.Dimensions;
         }
 
+        public void set_shape(Tensor shape)
+        {
+            this.shape = shape is null ? null : shape.shape;
+        }
+
         public int[] dims => shape;
 
         /// <summary>

src/TensorFlowNET.Core/Tensors/TensorShape.cs

@@ -9,6 +9,8 @@ namespace Tensorflow
     /// </summary>
     public class TensorShape : Shape
     {
+        public int[] dims => Dimensions;
+
         public TensorShape(TensorShapeProto proto)
         {
             if (proto.UnknownRank) return;
@@ -45,6 +47,29 @@ namespace Tensorflow
             throw new NotImplementedException("TensorShape is_compatible_with");
         }
 
+        public TensorShape with_rank_at_least(int rank)
+        {
+            if (rank != this.NDim)
+                throw new ValueError($"Shape {this} must have rank at least {rank}");
+            else
+                return this;
+        }
+
+        /// <summary>
+        /// Returns the concatenation of the dimension in `self` and `other`.
+        /// </summary>
+        /// <param name="other"></param>
+        /// <returns></returns>
+        public TensorShape concatenate(int[] other_)
+        {
+            var other = new TensorShape(other_);
+
+            if (NDim < 0 || other.NDim < 0)
+                return new TensorShape();
+            else
+                return new TensorShape(NDim + other.NDim);
+        }
+
         public static implicit operator TensorShape(int[] dims) => new TensorShape(dims);
         public static implicit operator TensorShape((int, int) dims) => new TensorShape(dims.Item1, dims.Item2);
         public static implicit operator TensorShape((int, int, int) dims) => new TensorShape(dims.Item1, dims.Item2, dims.Item3);

src/TensorFlowNET.Core/Util/nest.py.cs

@@ -223,31 +223,27 @@ namespace Tensorflow.Util
 
         private static void _flatten_recursive<T>(T obj, List<T> list)
         {
-            if (obj is string)
-            {
-                list.Add(obj);
-                return;
-            }
-            if (obj is IDictionary)
-            {
-                var dict = obj as IDictionary;
-                foreach (var key in _sorted(dict))
-                    _flatten_recursive((T)dict[key], list);
-                return;
-            }
-            if (obj is NDArray)
-            {
-                list.Add(obj);
-                return;
-            }
-            if (obj is IEnumerable)
+
+            switch(obj)
             {
-                var structure = obj as IEnumerable;
-                foreach (var child in structure)
-                    _flatten_recursive((T)child, list);
-                return;
+                case IDictionary dict:
+                    foreach (var key in _sorted(dict))
+                        _flatten_recursive((T)dict[key], list);
+                    break;
+                case String str:
+                    list.Add(obj);
+                    break;
+                case NDArray nd:
+                    list.Add(obj);
+                    break;
+                case IEnumerable structure:
+                    foreach (var child in structure)
+                        _flatten_recursive((T)child, list);
+                    break;
+                default:
+                    list.Add(obj);
+                    break;
             }
-            list.Add(obj);
         }
 
 

src/TensorFlowNET.Core/ops.py.cs

@@ -314,6 +314,11 @@ namespace Tensorflow
             return uid_number++;
         }
 
+        public static void colocate_with(bool ignore_existing = false)
+        {
+            _colocate_with_for_gradient(null, null, ignore_existing);
+        }
+
         public static void colocate_with(Operation op, bool ignore_existing = false)
         {
             _colocate_with_for_gradient(op, null, ignore_existing);

tensorflowlib/README.md

@@ -40,7 +40,7 @@ Before running verify you installed  CUDA and cuDNN
 
 https://www.tensorflow.org/install/source_windows
 
-pacman -S git patch unzip
+`pacman -S git patch unzip`
 
 1. Build static library
 

test/TensorFlowNET.Examples/ImageProcessing/DigitRecognitionCNN.cs

@@ -42,7 +42,7 @@ namespace TensorFlowNET.Examples.ImageProcess
         int n_channels = 1;
 
         // Hyper-parameters
-        int epochs = 10;
+        int epochs = 5; // accuracy > 98%
         int batch_size = 100;
         float learning_rate = 0.001f;
         Datasets<DataSetMnist> mnist;
@@ -84,7 +84,7 @@ namespace TensorFlowNET.Examples.ImageProcess
                 Test(sess);
             });
 
-            return loss_test < 0.09 && accuracy_test > 0.95;
+            return loss_test < 0.05 && accuracy_test > 0.98;
         }
 
         public Graph BuildGraph()

test/TensorFlowNET.UnitTest/Basics/AssignTests.cs

@@ -0,0 +1,36 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Tensorflow;
+using static Tensorflow.Python;
+
+namespace TensorFlowNET.UnitTest.Basics
+{
+    [TestClass]
+    public sealed class AssignTests
+    {
+        [Ignore("Not implemented")]
+        [TestMethod]
+        public void ShouldAssignVariable()
+        {
+            var raw_data = new[] { 1.0, 2.0, 8.0, -1.0, 0.0, 5.5, 6.0, 16.0 };
+            var expected = new[] { false, true, false, false, true, false, true };
+
+            var spike = tf.Variable(false);
+
+            spike.initializer.run();
+            foreach (var i in range(1, 2))
+            {
+                if (raw_data[i] - raw_data[i - 1] > 5d)
+                {
+                    var updater = tf.assign(spike, tf.constant(true));
+                    updater.eval();
+                }
+                else
+                {
+                    tf.assign(spike, tf.constant(true)).eval();
+                }
+
+                Assert.AreEqual((bool)spike.eval(), expected[i - 1]);
+            }
+        }
+    }
+}

test/TensorFlowNET.UnitTest/Basics/NegativeTests.cs

@@ -0,0 +1,24 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Tensorflow;
+using static Tensorflow.Python;
+
+namespace TensorFlowNET.UnitTest.Basics
+{
+    [TestClass]
+    public sealed class NegativeTests
+    {
+        [TestMethod]
+        public void ShouldReturnNegative()
+        {
+            var x = tf.constant(new[,] { { 1, 2 } });
+            var neg_x = tf.negative(x);
+            with(tf.Session(), session =>
+            {
+                var result = session.run(neg_x);
+
+                Assert.AreEqual(result[0][0], -1);
+                Assert.AreEqual(result[0][1], -2);
+            });
+        }
+    }
+}

test/TensorFlowNET.UnitTest/ExamplesTests/ExamplesTest.cs

@@ -106,6 +106,7 @@ namespace TensorFlowNET.ExamplesTests
             Assert.IsTrue(new NeuralNetXor() { Enabled = true, IsImportingGraph = false }.Run());
         }
 
+        [Ignore("Not working")]
         [TestMethod]
         public void NeuralNetXor_ImportedGraph()
         {
@@ -113,7 +114,7 @@ namespace TensorFlowNET.ExamplesTests
             Assert.IsTrue(new NeuralNetXor() { Enabled = true, IsImportingGraph = true }.Run());
         }
 
-
+        [Ignore("Not working")]
         [TestMethod]
         public void ObjectDetection()
         {

test/TensorFlowNET.UnitTest/Hub/MnistModelLoaderTest.cs

@@ -1,9 +1,5 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-using System.Collections.Generic;
-using System.Text;
 using System.Threading.Tasks;
-using Tensorflow;
 using Tensorflow.Hub;
 
 namespace TensorFlowNET.UnitTest.Hub

test/TensorFlowNET.UnitTest/OperationsTest.cs

@@ -89,6 +89,47 @@ namespace TensorFlowNET.UnitTest
             }
         }
 
+        [TestMethod]
+        public void cumSumTest()
+        {
+            var a = tf.constant(new[] { 1, 1, 2, 3, 4, 5 });
+            var b = tf.cumsum(a);
+            var check = np.array(1, 2, 4, 7, 11, 16);
+
+            using (var sess = tf.Session())
+            {
+                var o = sess.run(b);
+                Assert.IsTrue(o.array_equal(check));
+            }
+
+            b = tf.cumsum(a, exclusive: true);
+            check = np.array(0, 1, 2, 4, 7, 11);
+
+            using (var sess = tf.Session())
+            {
+                var o = sess.run(b);
+                Assert.IsTrue(o.array_equal(check));
+            }
+
+            b = tf.cumsum(a, reverse: true);
+            check = np.array(16, 15, 14, 12, 9, 5);
+
+            using (var sess = tf.Session())
+            {
+                var o = sess.run(b);
+                Assert.IsTrue(o.array_equal(check));
+            }
+
+            b = tf.cumsum(a, exclusive:true, reverse: true);
+            check = np.array(15, 14, 12, 9, 5, 0);
+
+            using (var sess = tf.Session())
+            {
+                var o = sess.run(b);
+                Assert.IsTrue(o.array_equal(check));
+            }
+        }
+
         [TestMethod]
         public void addOpTests()
         {