Merge branch 'master' of https://github.com/SciSharp/TensorFlow.NET

TensorFlow.NET.sln

@@ -9,8 +9,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Examples", "t
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Core", "src\TensorFlowNET.Core\TensorFlowNET.Core.csproj", "{FD682AC0-7B2D-45D3-8B0D-C6D678B04144}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Keras.Core", "src\KerasNET.Core\Keras.Core.csproj", "{902E188F-A953-43B4-9991-72BAB1697BC3}"
-EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TensorFlowNET.Examples.FSharp", "test\TensorFlowNET.Examples.FSharp\TensorFlowNET.Examples.FSharp.fsproj", "{62BC3801-F0D3-44A9-A0AC-712F40C8F961}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowBenchmark", "src\TensorFlowNet.Benchmarks\TensorFlowBenchmark.csproj", "{68861442-971A-4196-876E-C9330F0B3C54}"
@@ -41,10 +39,6 @@ Global
 		{FD682AC0-7B2D-45D3-8B0D-C6D678B04144}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FD682AC0-7B2D-45D3-8B0D-C6D678B04144}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{FD682AC0-7B2D-45D3-8B0D-C6D678B04144}.Release|Any CPU.Build.0 = Release|Any CPU
-		{902E188F-A953-43B4-9991-72BAB1697BC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{902E188F-A953-43B4-9991-72BAB1697BC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{902E188F-A953-43B4-9991-72BAB1697BC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{902E188F-A953-43B4-9991-72BAB1697BC3}.Release|Any CPU.Build.0 = Release|Any CPU
 		{62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{62BC3801-F0D3-44A9-A0AC-712F40C8F961}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/TensorFlowNET.Core/Binding.Util.cs

@@ -277,39 +277,6 @@ namespace Tensorflow
             return (__memberobject__.Length > 0) ? true : false;
         }
 
-        public delegate object __object__(params object[] args);
-
-        public static __object__ getattr(object obj, string key, params Type[] ___parameter_type__)
-        {
-            var __dyn_obj__ = obj.GetType().GetMember(key);
-            if (__dyn_obj__.Length == 0)
-                throw new Exception("The object \"" + nameof(obj) + "\" doesnot have a defination \"" + key + "\"");
-            var __type__ = __dyn_obj__[0];
-            if (__type__.MemberType == System.Reflection.MemberTypes.Method)
-            {
-                try
-                {
-                    var __method__ = (___parameter_type__.Length > 0) ? obj.GetType().GetMethod(key, ___parameter_type__) : obj.GetType().GetMethod(key);
-                    return (object[] args) => __method__.Invoke(obj, args);
-                }
-                catch (System.Reflection.AmbiguousMatchException ex)
-                {
-                    throw new Exception("AmbigousFunctionMatchFound : (Probable cause : Function Overloading) Please add parameter types of the function.");
-                }
-            }
-            else if (__type__.MemberType == System.Reflection.MemberTypes.Field)
-            {
-                var __field__ = obj.GetType().GetField(key).GetValue(obj);
-                return (object[] args) => { return __field__; };
-            }
-            else if (__type__.MemberType == System.Reflection.MemberTypes.Property)
-            {
-                var __property__ = obj.GetType().GetProperty(key).GetValue(obj);
-                return (object[] args) => { return __property__; };
-            }
-            return (object[] args) => { return "NaN"; };
-        }
-
         public static IEnumerable TupleToEnumerable(object tuple)
         {
             Type t = tuple.GetType();

src/TensorFlowNET.Core/Graphs/Graph.cs

@@ -420,7 +420,20 @@ namespace Tensorflow
 
         public List<T> get_collection<T>(string name, string scope = null)
         {
-            return _collections.ContainsKey(name) ? _collections[name] as List<T> : new List<T>();
+            List<T> t = default;
+            var collection = _collections.ContainsKey(name) ? _collections[name] : new List<T>();
+            switch (collection)
+            {
+                case List<VariableV1> list:
+                    t = list.Select(x => (T)(object)x).ToList();
+                    break;
+                case List<RefVariable> list:
+                    t = list.Select(x => (T)(object)x).ToList();
+                    break;
+                default:
+                    throw new NotImplementedException($"get_collection<{typeof(T).FullName}>");
+            }
+            return t;
         }
 
         public object get_collection_ref(string name)

src/TensorFlowNET.Core/Operations/Operation.Output.cs

@@ -17,13 +17,14 @@
 using System;
 using System.Linq;
 using System.Runtime.InteropServices;
+using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
     public partial class Operation
     {
         public int NumOutputs => c_api.TF_OperationNumOutputs(_handle);
-        public TF_DataType OutputType(int index) => c_api.TF_OperationOutputType(new TF_Output(_handle, index));
+        public TF_DataType OutputType(int index) => c_api.TF_OperationOutputType(_tf_output(index));
 
         public int OutputListLength(string name)
         {
@@ -48,6 +49,20 @@ namespace Tensorflow
 
         public TF_Output this[int index] => _tf_output(index);
 
+        /// <summary>
+        /// List this operation's output types.
+        /// </summary>
+        public TF_DataType[] _output_types
+        {
+            get
+            {
+                var output_types = range(NumOutputs)
+                    .Select(i => OutputType(i))
+                    .ToArray();
+                return output_types;
+            }
+        }
+
         public unsafe TF_Input[] OutputConsumers(int index, int max_consumers)
         {
             var handle = Marshal.AllocHGlobal(Marshal.SizeOf<TF_Input>());

src/TensorFlowNET.Core/Operations/Operation.cs

@@ -44,7 +44,6 @@ namespace Tensorflow
     public partial class Operation : ITensorOrOperation
     {
         private readonly IntPtr _handle; // _c_op in python
-        private readonly IntPtr _operDesc;
         private readonly Graph _graph;
         private NodeDef _node_def;
 
@@ -91,7 +90,7 @@ namespace Tensorflow
         {
             _graph = g;
 
-            _operDesc = c_api.TF_NewOperation(g, opType, oper_name);
+            var _operDesc = c_api.TF_NewOperation(g, opType, oper_name);
             c_api.TF_SetAttrType(_operDesc, "dtype", TF_DataType.TF_INT32);
             lock (Locks.ProcessWide)
                 using (var status = new Status())
@@ -161,7 +160,7 @@ namespace Tensorflow
                 op_def = g.GetOpDef(node_def.Op);
 
             var grouped_inputs = _reconstruct_sequence_inputs(op_def, inputs, node_def.Attr);
-            (_handle, _operDesc) = ops._create_c_op(g, node_def, grouped_inputs, control_input_ops.ToArray());
+            _handle = ops._create_c_op(g, node_def, grouped_inputs, control_input_ops.ToArray());
 
             // Initialize self._outputs.
             output_types = new TF_DataType[NumOutputs];
@@ -170,7 +169,7 @@ namespace Tensorflow
 
             _outputs = new Tensor[NumOutputs];
             for (int i = 0; i < NumOutputs; i++)
-                _outputs[i] = new Tensor(this, i, OutputType(i));
+                _outputs[i] = new Tensor(this, i, output_types[i]);
 
             graph._add_op(this);
 

src/TensorFlowNET.Core/Operations/c_api.ops.cs

@@ -198,7 +198,7 @@ namespace Tensorflow
         /// <param name="max_consumers">int</param>
         /// <returns></returns>
         [DllImport(TensorFlowLibName)]
-        public static extern unsafe int TF_OperationOutputConsumers(TF_Output oper_out, IntPtr consumers, int max_consumers);
+        public static extern int TF_OperationOutputConsumers(TF_Output oper_out, IntPtr consumers, int max_consumers);
 
         [DllImport(TensorFlowLibName)]
         public static extern TF_DataType TF_OperationOutputType(TF_Output oper_out);

src/TensorFlowNET.Core/Operations/control_flow_ops.py.cs

@@ -275,7 +275,7 @@ namespace Tensorflow
         /// </returns>
         public static Tensor[] _SwitchRefOrTensor(Tensor data, Tensor pred, string name = "Switch")
         {
-            data = ops.convert_to_tensor_or_indexed_slices(data, name: "data");
+            data = ops.convert_to_tensor_or_composite(data, name: "data");
             // NOTE(vrv): ops.colocate_with(data, ignore_existing=True) below
             // addresses the following scenario.
             //
@@ -296,9 +296,8 @@ namespace Tensorflow
             {
                 if (data is Tensor)
                 {
-                    // TODO: ref_switch
-                    //if (data.dtype._is_ref_dtype)
-                    //    return control_flow_ops.ref_switch(data, pred, name = name);
+                    if (data.dtype.is_ref_dtype())
+                        return gen_control_flow_ops.ref_switch(data, pred, name: name);
                 }
                 return @switch(data, pred, name: name);
             }            

src/TensorFlowNET.Core/Operations/gen_control_flow_ops.py.cs

@@ -114,6 +114,12 @@ namespace Tensorflow
             return _op;
         }
 
+        public static Tensor[] ref_switch(Tensor data, Tensor pred, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("RefSwitch", name, new { data, pred });
+            return _op.outputs;
+        }
+
         /// <summary>
         /// Forwards `data` to the output port determined by `pred`.
         /// 

src/TensorFlowNET.Core/TensorFlowNET.Core.csproj

@@ -5,7 +5,7 @@
     <AssemblyName>TensorFlow.NET</AssemblyName>
     <RootNamespace>Tensorflow</RootNamespace>
     <TargetTensorFlow>1.14.0</TargetTensorFlow>
-    <Version>0.11.2</Version>
+    <Version>0.11.3</Version>
     <Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
     <Company>SciSharp STACK</Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
@@ -17,7 +17,7 @@
     <PackageTags>TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET, C#</PackageTags>
     <Description>Google's TensorFlow full binding in .NET Standard.
 Docs: https://tensorflownet.readthedocs.io</Description>
-    <AssemblyVersion>0.11.2.0</AssemblyVersion>
+    <AssemblyVersion>0.11.3.0</AssemblyVersion>
     <PackageReleaseNotes>Changes since v0.10.0:
 1. Upgrade NumSharp to v0.20.
 2. Add DisposableObject class to manage object lifetime.
@@ -30,7 +30,7 @@ Docs: https://tensorflownet.readthedocs.io</Description>
 9. MultiThread is safe.
 10. Support n-dim indexing for tensor.</PackageReleaseNotes>
     <LangVersion>7.3</LangVersion>
-    <FileVersion>0.11.2.0</FileVersion>
+    <FileVersion>0.11.3.0</FileVersion>
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
     <SignAssembly>true</SignAssembly>

src/TensorFlowNET.Core/Train/ExponentialMovingAverage.cs

@@ -13,7 +13,7 @@ namespace Tensorflow.Train
         bool _zero_debias;
         string _name;
         public string name => _name;
-        List<VariableV1> _averages;
+        Dictionary<RefVariable, RefVariable> _averages;
 
         public ExponentialMovingAverage(float decay, int? num_updates = null, bool zero_debias = false,
             string name = "ExponentialMovingAverage")
@@ -22,7 +22,7 @@ namespace Tensorflow.Train
             _num_updates = num_updates;
             _zero_debias = zero_debias;
             _name = name;
-            _averages = new List<VariableV1>();
+            _averages = new Dictionary<RefVariable, RefVariable>();
         }
 
         /// <summary>
@@ -37,16 +37,38 @@ namespace Tensorflow.Train
 
             foreach(var var in var_list)
             {
-                if (!_averages.Contains(var))
+                if (!_averages.ContainsKey(var))
                 {
                     ops.init_scope();
-                    var slot = new SlotCreator();
-                    var.initialized_value();
-                    // var avg = slot.create_zeros_slot
+                    var slot_creator = new SlotCreator();
+                    var value = var.initialized_value();
+                    var avg = slot_creator.create_slot(var,
+                        value,
+                        name,
+                        colocate_with_primary: true);
+                    ops.add_to_collection(ops.GraphKeys.MOVING_AVERAGE_VARIABLES, var);
+                    _averages[var] = avg;
                 }
             }
 
-            throw new NotImplementedException("");
+            return tf_with(ops.name_scope(name), scope =>
+            {
+                var decay = ops.convert_to_tensor(_decay, name: "decay");
+                if (_num_updates.HasValue)
+                {
+                    throw new NotImplementedException("ExponentialMovingAverage.apply");
+                }
+
+                var updates = new List<Tensor>();
+                foreach (var var in var_list)
+                {
+                    var zero_debias = false;// _averages[var] in zero_debias_true
+                    var ama = moving_averages.assign_moving_average(_averages[var], var, decay, zero_debias: zero_debias);
+                    updates.Add(ama);
+                }
+
+                return control_flow_ops.group(updates.ToArray(), name: scope);
+            });
         }
     }
 }

src/TensorFlowNET.Core/Train/SlotCreator.cs

@@ -22,6 +22,24 @@ namespace Tensorflow.Train
 {
     public class SlotCreator
     {
+        /// <summary>
+        /// Create a slot initialized to the given value.
+        /// </summary>
+        /// <param name="primary"></param>
+        /// <param name="val"></param>
+        /// <param name="name"></param>
+        /// <param name="colocate_with_primary"></param>
+        /// <returns></returns>
+        public RefVariable create_slot(RefVariable primary, Tensor val, string name, bool colocate_with_primary = true)
+        {
+            var validate_shape = val.TensorShape.is_fully_defined();
+            var prefix = primary.op.name;
+            return tf_with(tf.variable_scope(name: null, prefix + "/" + name), delegate
+            {
+                return _create_slot_var(primary, val, "", validate_shape, null, TF_DataType.DtInvalid);
+            });
+        }
+
         /// <summary>
         /// Create a slot initialized to 0 with same shape as the primary object.
         /// </summary>
@@ -73,7 +91,7 @@ namespace Tensorflow.Train
         /// <param name="shape"></param>
         /// <param name="dtype"></param>
         /// <returns></returns>
-        private RefVariable _create_slot_var(VariableV1 primary, IInitializer val, string scope, bool validate_shape, 
+        private RefVariable _create_slot_var(VariableV1 primary, object val, string scope, bool validate_shape, 
             TensorShape shape, TF_DataType dtype)
         {
             bool use_resource = primary is ResourceVariable;

src/TensorFlowNET.Core/Train/moving_averages.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Train
+{
+    public class moving_averages
+    {
+        /// <summary>
+        /// Compute the moving average of a variable.
+        /// </summary>
+        /// <param name="variable"></param>
+        /// <param name="value"></param>
+        /// <param name="decay"></param>
+        /// <param name="zero_debias"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor assign_moving_average(RefVariable variable, RefVariable value, Tensor decay,
+            bool zero_debias = true, string name = null)
+        {
+            tf_with(ops.name_scope(name, "", new { variable, value, decay }), scope =>
+            {
+                decay = ops.convert_to_tensor(1.0f - decay, name: "decay");
+                if (decay.dtype != variable.dtype.as_base_dtype())
+                    decay = math_ops.cast(decay, variable.dtype.as_base_dtype());
+            });
+
+            throw new NotImplementedException("assign_moving_average");
+        }
+    }
+}

src/TensorFlowNET.Core/Variables/RefVariable.cs

@@ -17,6 +17,7 @@
 using Google.Protobuf;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using static Tensorflow.Binding;
 
 namespace Tensorflow
@@ -176,7 +177,7 @@ namespace Tensorflow
                 // If 'initial_value' makes use of other variables, make sure we don't
                 // have an issue if these other variables aren't initialized first by
                 // using their initialized_value() method.
-                var _initial_value2 = _try_guard_against_uninitialized_dependencies(_initial_value);
+                var _initial_value2 = _try_guard_against_uninitialized_dependencies(name, _initial_value);
 
                 _initializer_op = gen_state_ops.assign(_variable, _initial_value2, validate_shape).op;
 
@@ -215,9 +216,9 @@ namespace Tensorflow
         /// Attempt to guard against dependencies on uninitialized variables.
         /// </summary>
         /// <param name="initial_value"></param>
-        private Tensor _try_guard_against_uninitialized_dependencies(Tensor initial_value)
+        private Tensor _try_guard_against_uninitialized_dependencies(string name, Tensor initial_value)
         {
-            return _safe_initial_value_from_tensor(initial_value, new Dictionary<string, Operation>());
+            return _safe_initial_value_from_tensor(name, initial_value, op_cache: new Dictionary<string, Operation>());
         }
 
         /// <summary>
@@ -226,19 +227,19 @@ namespace Tensorflow
         /// <param name="tensor">A `Tensor`. The tensor to replace.</param>
         /// <param name="op_cache">A dict mapping operation names to `Operation`s.</param>
         /// <returns>A `Tensor` compatible with `tensor`.</returns>
-        private Tensor _safe_initial_value_from_tensor(Tensor tensor, Dictionary<string, Operation> op_cache)
+        private Tensor _safe_initial_value_from_tensor(string name, Tensor tensor, Dictionary<string, Operation> op_cache)
         {
             var op = tensor.op;
             var new_op = op_cache.ContainsKey(op.name) ? op_cache[op.name] : null;
             if(new_op == null)
             {
-                new_op = _safe_initial_value_from_op(op, op_cache);
+                new_op = _safe_initial_value_from_op(name, op, op_cache);
                 op_cache[op.name] = new_op;
             }
             return new_op.outputs[tensor.value_index];
         }
 
-        private Operation _safe_initial_value_from_op(Operation op, Dictionary<string, Operation> op_cache)
+        private Operation _safe_initial_value_from_op(string name, Operation op, Dictionary<string, Operation> op_cache)
         {
             var op_type = op.node_def.Op;
             switch (op_type)
@@ -250,13 +251,50 @@ namespace Tensorflow
                 case "Variable":
                 case "VariableV2":
                 case "VarHandleOp":
-                    break;
+                    var initialized_value = _find_initialized_value_for_variable(op);
+                    return initialized_value == null ? op : initialized_value.op;
             }
 
             // Recursively build initializer expressions for inputs.
+            var modified = false;
+            var new_op_inputs = new List<Tensor>();
+            foreach (var op_input in op.inputs)
+            {
+                var new_op_input = _safe_initial_value_from_tensor(name, op_input as Tensor, op_cache);
+                new_op_inputs.Add(new_op_input);
+                modified = modified || new_op_input != op_input;
+            }
+
+            // If at least one input was modified, replace the op.
+            if (modified)
+            {
+                var new_op_type = op_type;
+                if (new_op_type == "RefSwitch")
+                    new_op_type = "Switch";
+                var new_op_name = op.node_def.Name + "_" + name;
+                new_op_name = new_op_name.Replace(":", "_");
+                var attrs = new Dictionary<string, AttrValue>();
+                attrs[op.node_def.Name] = op.node_def.Attr.ElementAt(0).Value;
+                /*return op.graph.create_op(new_op_type, new_op_inputs.ToArray(), op._output_types, 
+                    name: new_op_name, attrs: attrs);*/
+            }
             return op;
         }
 
+        private Operation _find_initialized_value_for_variable(Operation variable_op)
+        {
+            var var_names = new[] { variable_op.node_def.Name, variable_op.node_def.Name + ":0" };
+            foreach(var collection_name in new[]{tf.GraphKeys.GLOBAL_VARIABLES,
+                            tf.GraphKeys.LOCAL_VARIABLES })
+            {
+                foreach (var var in variable_op.graph.get_collection<RefVariable>(collection_name))
+                    if (var_names.Contains(var.name))
+                        return var.initialized_value();
+            }
+
+           return null;
+        }
+
         /// <summary>
         /// Assigns a new value to the variable.
         /// </summary>
@@ -318,6 +356,15 @@ namespace Tensorflow
             return array_ops.identity(_variable, name: "read");
         }
 
+        /// <summary>
+        /// Returns the Tensor used as the initial value for the variable.
+        /// </summary>
+        /// <returns></returns>
+        private ITensorOrOperation initial_value()
+        {
+            return _initial_value;
+        }
+
         public Tensor is_variable_initialized(RefVariable variable)
         {
             return state_ops.is_variable_initialized(variable);
@@ -326,10 +373,9 @@ namespace Tensorflow
         public Tensor initialized_value()
         {
             ops.init_scope();
-            throw new NotImplementedException("");
-            /*return control_flow_ops.cond(is_variable_initialized(this),
+            return control_flow_ops.cond(is_variable_initialized(this),
                                    read_value,
-                                   () => initial_value);*/
+                                   initial_value);
         }
     }
 }

src/TensorFlowNET.Core/Variables/gen_state_ops.py.cs

@@ -149,7 +149,8 @@ namespace Tensorflow
 
         public static Tensor is_variable_initialized(RefVariable @ref, string name = null)
         {
-            throw new NotImplementedException("");
+            var _op = _op_def_lib._apply_op_helper("IsVariableInitialized", name: name, args: new { @ref });
+            return _op.output;
         }
     }
 }

src/TensorFlowNET.Core/ops.GraphKeys.cs

@@ -52,6 +52,8 @@ namespace Tensorflow
             /// </summary>
             public const string LOSSES_ = "losses";
 
+            public const string MOVING_AVERAGE_VARIABLES = "moving_average_variables";
+
             /// <summary>
             /// Key to collect Variable objects that are global (shared across machines).
             /// Default collection for all variables, except local ones.
@@ -100,6 +102,12 @@ namespace Tensorflow
             /// </summary>
             public string _STREAMING_MODEL_PORTS => _STREAMING_MODEL_PORTS_;
 
+            /// <summary>
+            /// Key to collect local variables that are local to the machine and are not
+            /// saved/restored.
+            /// </summary>
+            public string LOCAL_VARIABLES = "local_variables";
+
             /// <summary>
             /// Key to collect losses
             /// </summary>

src/TensorFlowNET.Core/ops.cs

@@ -206,7 +206,7 @@ namespace Tensorflow
         /// </param>
         /// <param name="control_inputs">A list of `Operation`s to set as control dependencies.</param>
         /// <returns>A wrapped TF_Operation*.</returns>
-        public static (IntPtr, IntPtr) _create_c_op<T>(Graph graph, NodeDef node_def, T[] inputs, Operation[] control_inputs)
+        public static IntPtr _create_c_op<T>(Graph graph, NodeDef node_def, T[] inputs, Operation[] control_inputs)
         {
             lock (Locks.ProcessWide)
             {
@@ -249,7 +249,7 @@ namespace Tensorflow
 
                 status.Check(true);
 
-                return (c_op, op_desc);
+                return c_op;
             }
         }
 

test/TensorFlowNET.Examples/ImageProcessing/YOLO/Main.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using Tensorflow;
 using static Tensorflow.Binding;
@@ -47,6 +48,9 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
         YOLOv3 model;
         VariableV1[] net_var;
         Tensor giou_loss, conf_loss, prob_loss;
+        RefVariable global_step;
+        Tensor learn_rate;
+        Tensor loss;
         #endregion
 
         public bool Run()
@@ -98,11 +102,45 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
                 (giou_loss, conf_loss, prob_loss) = model.compute_loss(
                                                     label_sbbox, label_mbbox, label_lbbox,
                                                     true_sbboxes, true_mbboxes, true_lbboxes);
+                loss = giou_loss + conf_loss + prob_loss;
             });
 
+            Tensor global_step_update = null;
+            tf_with(tf.name_scope("learn_rate"), scope =>
+            {
+                global_step = tf.Variable(1.0, dtype: tf.float64, trainable: false, name: "global_step");
+                var warmup_steps = tf.constant(warmup_periods * steps_per_period,
+                                        dtype: tf.float64, name: "warmup_steps");
+                var train_steps = tf.constant((first_stage_epochs + second_stage_epochs) * steps_per_period,
+                                        dtype: tf.float64, name: "train_steps");
+
+                learn_rate = tf.cond(
+                    pred: global_step < warmup_steps,
+                    true_fn: delegate
+                    {
+                        return global_step / warmup_steps * learn_rate_init;
+                    },
+                    false_fn: delegate
+                    {
+                        return learn_rate_end + 0.5 * (learn_rate_init - learn_rate_end) *
+                            (1 + tf.cos(
+                                (global_step - warmup_steps) / (train_steps - warmup_steps) * Math.PI));
+                    }
+                );
+
+                global_step_update = tf.assign_add(global_step, 1.0f);
+            });
+
+            Operation moving_ave = null;
             tf_with(tf.name_scope("define_weight_decay"), scope =>
             {
-                var moving_ave = tf.train.ExponentialMovingAverage(moving_ave_decay).apply((RefVariable[])tf.trainable_variables());
+                var emv = tf.train.ExponentialMovingAverage(moving_ave_decay);
+                var vars = tf.trainable_variables().Select(x => (RefVariable)x).ToArray();
+                moving_ave = emv.apply(vars);
+            });
+
+            tf_with(tf.name_scope("define_first_stage_train"), scope =>
+            {
             });
 
             return graph;

test/TensorFlowNET.Examples/ImageProcessing/YOLO/YOLOv3.cs

@@ -23,6 +23,8 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
         Tensor conv_mbbox;
         Tensor conv_sbbox;
         Tensor pred_sbbox;
+        Tensor pred_mbbox;
+        Tensor pred_lbbox;
 
         public YOLOv3(Config cfg_, Tensor input_data_, Tensor trainable_)
         {
@@ -46,12 +48,12 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
 
             tf_with(tf.variable_scope("pred_mbbox"), scope =>
             {
-                pred_sbbox = decode(conv_sbbox, anchors[0], strides[0]);
+                pred_mbbox = decode(conv_mbbox, anchors[1], strides[1]);
             });
 
             tf_with(tf.variable_scope("pred_lbbox"), scope =>
             {
-                pred_sbbox = decode(conv_sbbox, anchors[0], strides[0]);
+                pred_lbbox = decode(conv_lbbox, anchors[2], strides[2]);
             });
         }
 
@@ -144,6 +146,8 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
         {
             Tensor giou_loss = null, conf_loss = null, prob_loss = null;
             (Tensor, Tensor, Tensor) loss_sbbox = (null, null, null);
+            (Tensor, Tensor, Tensor) loss_mbbox = (null, null, null);
+            (Tensor, Tensor, Tensor) loss_lbbox = (null, null, null);
 
             tf_with(tf.name_scope("smaller_box_loss"), delegate
             {
@@ -151,6 +155,33 @@ namespace TensorFlowNET.Examples.ImageProcessing.YOLO
                                          anchors: anchors[0], stride: strides[0]);
             });
 
+            tf_with(tf.name_scope("medium_box_loss"), delegate
+            {
+                loss_mbbox = loss_layer(conv_mbbox, pred_mbbox, label_mbbox, true_mbbox,
+                                         anchors: anchors[1], stride: strides[1]);
+            });
+
+            tf_with(tf.name_scope("bigger_box_loss"), delegate
+            {
+                loss_lbbox = loss_layer(conv_lbbox, pred_lbbox, label_lbbox, true_lbbox,
+                                         anchors: anchors[2], stride: strides[2]);
+            });
+
+            tf_with(tf.name_scope("giou_loss"), delegate
+            {
+                giou_loss = loss_sbbox.Item1 + loss_mbbox.Item1 + loss_lbbox.Item1;
+            });
+
+            tf_with(tf.name_scope("conf_loss"), delegate
+            {
+                conf_loss = loss_sbbox.Item2 + loss_mbbox.Item2 + loss_lbbox.Item2;
+            });
+
+            tf_with(tf.name_scope("prob_loss"), delegate
+            {
+                prob_loss = loss_sbbox.Item3 + loss_mbbox.Item3 + loss_lbbox.Item3;
+            });
+
             return (giou_loss, conf_loss, prob_loss);
         }
 

test/TensorFlowNET.Examples/TensorFlowNET.Examples.GPU.csproj

@@ -14,6 +14,10 @@
     <OutputPath>bin\release-gpu</OutputPath>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="Keras.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Colorful.Console" Version="1.2.9" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
@@ -23,7 +27,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\src\KerasNET.Core\Keras.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowDatasets\TensorFlowDatasets.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\TensorFlowNET.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowText\TensorFlowText.csproj" />

test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj

@@ -10,6 +10,10 @@
     <DefineConstants>DEBUG;TRACE</DefineConstants>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="Keras.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Colorful.Console" Version="1.2.9" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
@@ -19,7 +23,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\src\KerasNET.Core\Keras.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowDatasets\TensorFlowDatasets.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\TensorFlowNET.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowText\TensorFlowText.csproj" />

test/TensorFlowNET.UnitTest/PythonBaseTests.cs

@@ -49,27 +49,6 @@ namespace TensorFlowNET.UnitTest
             Assert.IsFalse(false2);
             Assert.IsFalse(false3);
         }
-
-        [TestMethod]
-        public void hasattr_getattr()
-        {
-            var s1 = "Tensorflow v0.1";
-            var f = "Tensorflow";
-            var r = "Tensorflow.NET";
-            var res = s1.Replace(f, r);
-
-            // Test 1
-            Assert.IsTrue(hasattr(s1, "Replace"));
-
-            // Test 2
-            var o = getattr( s1, "Replace", typeof(string), typeof(string));
-            Assert.AreEqual(res, o(f, r));
-
-            // Test 3
-            var l = getattr(s1, "Length");
-            Assert.AreEqual(s1.Length, l());
-
-        }
     }
 }
 

test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj

@@ -23,6 +23,10 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Remove="KerasTests.cs" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="5.9.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
@@ -32,7 +36,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\src\KerasNET.Core\Keras.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowHub\TensorFlowHub.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\TensorFlowNET.Core.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowText\TensorFlowText.csproj" />

test/TensorFlowNET.UnitTest/ops_test/CreateOpFromTfOperationTest.cs

@@ -27,7 +27,7 @@ namespace TensorFlowNET.UnitTest.ops_test
             using (var g = tf.Graph().as_default())
             {
                 var x = constant_op.constant(new[,] {{1, 2, 3}, {4, 5, 6}});
-                var (c_op, op_desc) = ops._create_c_op(g, ops._NodeDef("Identity", "myop"), new[] {x}, new Operation[0]);
+                var c_op = ops._create_c_op(g, ops._NodeDef("Identity", "myop"), new[] {x}, new Operation[0]);
                 var op = g._create_op_from_tf_operation(c_op);
 
                 Assert.AreEqual("myop", op.name);
@@ -68,7 +68,7 @@ namespace TensorFlowNET.UnitTest.ops_test
 
             var true_fn = new Func<Tensor>(() =>
             {
-                var (c_op, op_desc) = ops._create_c_op(g, ops._NodeDef("Identity", "cond/myop"), new[] { x }, new Operation[0]);
+                var c_op = ops._create_c_op(g, ops._NodeDef("Identity", "cond/myop"), new[] { x }, new Operation[0]);
                 var new_ops = g._add_new_tf_operations();
                 self.assertEqual(len(new_ops), 1);
                 return x;