diff --git a/src/TensorFlowNET.Core/APIs/tf.init.cs b/src/TensorFlowNET.Core/APIs/tf.init.cs index 584f3b01..5b80313c 100644 --- a/src/TensorFlowNET.Core/APIs/tf.init.cs +++ b/src/TensorFlowNET.Core/APIs/tf.init.cs @@ -7,7 +7,12 @@ namespace Tensorflow public static partial class tf { public static IInitializer zeros_initializer => new Zeros(); + public static IInitializer glorot_uniform => new GlorotUniform(); + public static variable_scope variable_scope(string name_or_scope, + string default_name = null, + object values = null) => new variable_scope(name_or_scope, default_name, values); + public class Zeros : IInitializer { private TF_DataType dtype; @@ -30,5 +35,105 @@ namespace Tensorflow return new { dtype = dtype.name() }; } } + + /// + /// Initializer capable of adapting its scale to the shape of weights tensors. + /// + public class VarianceScaling : IInitializer + { + protected float _scale; + protected string _mode; + protected string _distribution; + protected int? _seed; + protected TF_DataType _dtype; + + public VarianceScaling(float scale = 1.0f, + string mode = "fan_in", + string distribution= "truncated_normal", + int? seed = null, + TF_DataType dtype = TF_DataType.TF_FLOAT) + { + if (scale < 0) + throw new ValueError("`scale` must be positive float."); + _scale = scale; + _mode = mode; + _distribution = distribution; + _seed = seed; + _dtype = dtype; + } + + public Tensor call(TensorShape shape, TF_DataType dtype) + { + var (fan_in, fan_out) = _compute_fans(shape); + if (_mode == "fan_in") + _scale /= Math.Max(1, fan_in); + else if (_mode == "fan_out") + _scale /= Math.Max(1, fan_out); + else + _scale /= Math.Max(1, (fan_in + fan_out) / 2); + + if (_distribution == "normal" || _distribution == "truncated_normal") + { + throw new NotImplementedException("truncated_normal"); + } + else if(_distribution == "untruncated_normal") + { + throw new NotImplementedException("truncated_normal"); + } + else + { + var limit = Math.Sqrt(3.0f * _scale); + return random_ops.random_uniform(shape, (float)-limit, (float)limit, dtype, seed: _seed); + } + } + + private (int, int) _compute_fans(int[] shape) + { + if (shape.Length < 1) + return (1, 1); + if (shape.Length == 1) + return (shape[0], shape[0]); + if (shape.Length == 2) + return (shape[0], shape[1]); + else + throw new NotImplementedException("VarianceScaling._compute_fans"); + } + + public virtual object get_config() + { + return new + { + scale = _scale, + mode = _mode, + distribution = _distribution, + seed = _seed, + dtype = _dtype + }; + } + } + + public class GlorotUniform : VarianceScaling + { + public GlorotUniform(float scale = 1.0f, + string mode = "fan_avg", + string distribution = "uniform", + int? seed = null, + TF_DataType dtype = TF_DataType.TF_FLOAT) : base(scale, mode, distribution, seed, dtype) + { + + } + + public object get_config() + { + return new + { + scale = _scale, + mode = _mode, + distribution = _distribution, + seed = _seed, + dtype = _dtype + }; + } + } } } diff --git a/src/TensorFlowNET.Core/Operations/random_ops.py.cs b/src/TensorFlowNET.Core/Operations/random_ops.py.cs index 299ea3f7..eae27c58 100644 --- a/src/TensorFlowNET.Core/Operations/random_ops.py.cs +++ b/src/TensorFlowNET.Core/Operations/random_ops.py.cs @@ -4,8 +4,18 @@ using System.Text; namespace Tensorflow { - public class random_ops + public class random_ops : Python { + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// public static Tensor random_normal(int[] shape, float mean = 0.0f, float stddev = 1.0f, @@ -26,6 +36,30 @@ namespace Tensorflow }); } + /// + /// Outputs random values from a uniform distribution. + /// + /// + /// + /// + /// The type of the output + /// Used to create a random seed for the distribution. + /// A name for the operation + /// A tensor of the specified shape filled with random uniform values. + public static Tensor random_uniform(int[] shape, + float minval = 0, + float? maxval = null, + TF_DataType dtype = TF_DataType.TF_FLOAT, + int? seed = null, + string name = null) + { + return with(new ops.name_scope(name, "random_uniform", new { shape, minval, maxval }), scope => + { + name = scope; + return null; + }); + } + private static Tensor _ShapeTensor(int[] shape) { return ops.convert_to_tensor(shape, name: "shape"); diff --git a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj index c4e5891d..388adf3e 100644 --- a/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj +++ b/src/TensorFlowNET.Core/TensorFlowNET.Core.csproj @@ -4,7 +4,7 @@ netstandard2.0 TensorFlow.NET Tensorflow - 0.4.0 + 0.4.1 Haiping Chen SciSharp STACK true @@ -16,11 +16,11 @@ TensorFlow, NumSharp, SciSharp, MachineLearning, TensorFlow.NET Google's TensorFlow binding in .NET Standard. Docs: https://tensorflownet.readthedocs.io - 0.4.0.0 - Added Linear Regression example. - + 0.4.1.0 + Added ConfigProto to control CPU and GPU resource. +Fixed import name scope issue. 7.2 - 0.4.0.0 + 0.4.1.0 diff --git a/src/TensorFlowNET.Core/Variables/PureVariableScope.cs b/src/TensorFlowNET.Core/Variables/PureVariableScope.cs new file mode 100644 index 00000000..15401eea --- /dev/null +++ b/src/TensorFlowNET.Core/Variables/PureVariableScope.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow +{ + public class PureVariableScope : IPython + { + private string _name_or_scope; + private string _new_name; + private string _old_name_scope; + private bool _reuse; + private _VariableStore _var_store; + private VariableScope _old; + private _VariableScopeStore _var_scope_store; + private VariableScope variable_scope_object; + + public PureVariableScope(string name_or_scope, + string old_name_scope = null, + TF_DataType dtype = TF_DataType.DtInvalid) + { + _name_or_scope = name_or_scope; + _old_name_scope = old_name_scope; + _var_store = variable_scope._get_default_variable_store(); + _var_scope_store = variable_scope.get_variable_scope_store(); + } + + public void __enter__() + { + _old = _var_scope_store.current_scope; + _new_name = string.IsNullOrEmpty(_old.name) ? _name_or_scope : _old.name + "/" + _name_or_scope; + _reuse = _reuse || _old.resue; + string name_scope = _old_name_scope == null ? _name_or_scope : _old_name_scope; + + variable_scope_object = new VariableScope(_reuse, + name: _new_name, + name_scope: name_scope); + + _var_scope_store.open_variable_scope(_new_name); + _var_scope_store.current_scope = variable_scope_object; + } + + public void Dispose() + { + + } + + public void __exit__() + { + + } + + public static implicit operator VariableScope(PureVariableScope scope) + { + return scope.variable_scope_object; + } + } +} diff --git a/src/TensorFlowNET.Core/Variables/VariableScope.cs b/src/TensorFlowNET.Core/Variables/VariableScope.cs index c8d99036..d5e16504 100644 --- a/src/TensorFlowNET.Core/Variables/VariableScope.cs +++ b/src/TensorFlowNET.Core/Variables/VariableScope.cs @@ -4,17 +4,26 @@ using System.Text; namespace Tensorflow { + /// + /// Variable scope object to carry defaults to provide to `get_variable` + /// public class VariableScope { public bool use_resource { get; set; } - private _ReuseMode _reuse { get; set; } + private _ReuseMode _reuse; + public bool resue; - private object _regularizer; private TF_DataType _dtype; public string name { get; set; } + public string name_scope { get; set; } - public VariableScope(TF_DataType dtype = TF_DataType.TF_FLOAT) + public VariableScope(bool reuse, + string name = "", + string name_scope = "", + TF_DataType dtype = TF_DataType.TF_FLOAT) { + this.name = name; + this.name_scope = name_scope; _reuse = _ReuseMode.AUTO_REUSE; _dtype = dtype; } @@ -29,7 +38,7 @@ namespace Tensorflow VariableAggregation aggregation= VariableAggregation.NONE) { string full_name = !string.IsNullOrEmpty(this.name) ? this.name + "/" + name : name; - return Python.with(new ops.name_scope(""), scope => + return Python.with(new ops.name_scope(null), scope => { if (dtype == TF_DataType.DtInvalid) dtype = _dtype; diff --git a/src/TensorFlowNET.Core/Variables/_VariableScopeStore.cs b/src/TensorFlowNET.Core/Variables/_VariableScopeStore.cs index a7b3e3b5..a043bfe0 100644 --- a/src/TensorFlowNET.Core/Variables/_VariableScopeStore.cs +++ b/src/TensorFlowNET.Core/Variables/_VariableScopeStore.cs @@ -7,10 +7,20 @@ namespace Tensorflow public class _VariableScopeStore { public VariableScope current_scope { get; set; } + private Dictionary variable_scopes_count; public _VariableScopeStore() { - current_scope = new VariableScope(); + current_scope = new VariableScope(false); + variable_scopes_count = new Dictionary(); + } + + public void open_variable_scope(string scope_name) + { + if (variable_scopes_count.ContainsKey(scope_name)) + variable_scopes_count[scope_name] += 1; + else + variable_scopes_count[scope_name] = 1; } } } diff --git a/src/TensorFlowNET.Core/Variables/_VariableStore.cs b/src/TensorFlowNET.Core/Variables/_VariableStore.cs index 2c22f25c..873d33a6 100644 --- a/src/TensorFlowNET.Core/Variables/_VariableStore.cs +++ b/src/TensorFlowNET.Core/Variables/_VariableStore.cs @@ -87,6 +87,14 @@ namespace Tensorflow } Tensor init_val = null; + + // Create the tensor to initialize the variable with default value. + if (initializer == null) + { + if (dtype.is_floating()) + initializer = tf.glorot_uniform; + } + ops.init_scope(); { if (initializing_from_value) diff --git a/src/TensorFlowNET.Core/Variables/tf.variable.cs b/src/TensorFlowNET.Core/Variables/tf.variable.cs index 2e7eefec..b61b558e 100644 --- a/src/TensorFlowNET.Core/Variables/tf.variable.cs +++ b/src/TensorFlowNET.Core/Variables/tf.variable.cs @@ -20,8 +20,8 @@ namespace Tensorflow VariableSynchronization synchronization = VariableSynchronization.AUTO, VariableAggregation aggregation = VariableAggregation.NONE) { - var scope = variable_scope.get_variable_scope(); - var store = variable_scope._get_default_variable_store(); + var scope = Tensorflow.variable_scope.get_variable_scope(); + var store = Tensorflow.variable_scope._get_default_variable_store(); return scope.get_variable(store, name, shape: shape, diff --git a/src/TensorFlowNET.Core/Variables/variable_scope.py.cs b/src/TensorFlowNET.Core/Variables/variable_scope.py.cs index 56439db5..d59daa35 100644 --- a/src/TensorFlowNET.Core/Variables/variable_scope.py.cs +++ b/src/TensorFlowNET.Core/Variables/variable_scope.py.cs @@ -4,12 +4,59 @@ using System.Text; namespace Tensorflow { - public class variable_scope + public class variable_scope : IPython { public static string _VARSTORE_KEY = "__variable_store"; public static string _VARSCOPESTORE_KEY = "__varscope"; public static bool _DEFAULT_USE_RESOURCE = false; + private bool _use_resource; + public bool UseResource => _use_resource; + private string _name_or_scope; + private string _default_name; + private object _values; + private string _current_name_scope; + private PureVariableScope _cached_pure_variable_scope; + + public variable_scope(string name_or_scope, string default_name = "", object values = null) + { + _name_or_scope = name_or_scope; + _default_name = default_name; + _values = values; + _current_name_scope = null; + + _use_resource = false; + if (_default_name == null && _name_or_scope == null) + throw new TypeError("If default_name is None then name_or_scope is required"); + } + + public void __enter__() + { + _enter_scope_uncached(); + } + + public VariableScope _enter_scope_uncached() + { + ops.name_scope current_name_scope = null; + if(_name_or_scope != null) + { + var name_scope = _name_or_scope; + if (name_scope != null || current_name_scope != null) + current_name_scope = new ops.name_scope(name_scope); + current_name_scope.__enter__(); + string current_name_scope_name = current_name_scope; + _current_name_scope = current_name_scope; + string old_name_scope = current_name_scope_name; + var pure_variable_scope = new PureVariableScope(_name_or_scope, old_name_scope: old_name_scope); + pure_variable_scope.__enter__(); + VariableScope entered_pure_variable_scope = pure_variable_scope; + _cached_pure_variable_scope = pure_variable_scope; + return entered_pure_variable_scope; + } + + throw new NotImplementedException("_enter_scope_uncached"); + } + public static RefVariable default_variable_creator(object initial_value, string name = null, bool? trainable = null, @@ -101,5 +148,15 @@ namespace Tensorflow return trainable.Value; } + + public void __exit__() + { + + } + + public void Dispose() + { + + } } } diff --git a/src/TensorFlowNET.Core/tf.cs b/src/TensorFlowNET.Core/tf.cs index 119a884e..eeb415f5 100644 --- a/src/TensorFlowNET.Core/tf.cs +++ b/src/TensorFlowNET.Core/tf.cs @@ -20,7 +20,7 @@ namespace Tensorflow public static RefVariable Variable(T data, string name = null, TF_DataType dtype = TF_DataType.DtInvalid) { - return variable_scope.default_variable_creator(data, name: name, dtype: TF_DataType.DtInvalid); + return Tensorflow.variable_scope.default_variable_creator(data, name: name, dtype: TF_DataType.DtInvalid); } public static unsafe Tensor placeholder(TF_DataType dtype, TensorShape shape = null) diff --git a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj index 2ff58949..f495711c 100644 --- a/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/TensorFlowNET.UnitTest.csproj @@ -1,7 +1,7 @@ - netcoreapp2.1 + netcoreapp2.2 false diff --git a/test/TensorFlowNET.UnitTest/VariableTest.cs b/test/TensorFlowNET.UnitTest/VariableTest.cs index 88c65bd3..15773152 100644 --- a/test/TensorFlowNET.UnitTest/VariableTest.cs +++ b/test/TensorFlowNET.UnitTest/VariableTest.cs @@ -30,6 +30,18 @@ namespace TensorFlowNET.UnitTest var mammal2 = tf.Variable("Tiger"); } + [TestMethod] + public void SimpleScope() + { + with(tf.variable_scope("foo"), delegate + { + with(tf.variable_scope("bar"), delegate + { + var v = tf.get_variable("v", new TensorShape(1)); + }); + }); + } + [TestMethod] public void ScalarVar() {