From ec8bd2eb330642d39b62ce1d743ce805932ce08e Mon Sep 17 00:00:00 2001 From: Luc BOLOGNA Date: Thu, 1 Jun 2023 23:50:55 +0200 Subject: [PATCH 01/77] refacto: Standardize TensorFlowNET.Keras/Losses/ Smooth implementation --- .../Losses/BinaryCrossentropy.cs | 4 +- .../Losses/CategoricalCrossentropy.cs | 4 +- .../Losses/CosineSimilarity.cs | 40 ++++----- src/TensorFlowNET.Keras/Losses/Huber.cs | 53 +++++------ src/TensorFlowNET.Keras/Losses/LogCosh.cs | 37 ++++---- src/TensorFlowNET.Keras/Losses/Loss.cs | 90 +++++++++---------- .../Losses/LossFunctionWrapper.cs | 22 +++-- .../Losses/MeanAbsoluteError.cs | 29 +++--- .../Losses/MeanAbsolutePercentageError.cs | 31 +++---- .../Losses/MeanSquaredError.cs | 29 +++--- .../Losses/MeanSquaredLogarithmicError.cs | 49 +++++----- .../Losses/SigmoidFocalCrossEntropy.cs | 3 +- .../Losses/SparseCategoricalCrossentropy.cs | 62 ++++++------- 13 files changed, 200 insertions(+), 253 deletions(-) diff --git a/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs index ff7bb6b7..0de50a7e 100644 --- a/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs +++ b/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs @@ -1,8 +1,9 @@ namespace Tensorflow.Keras.Losses; -public class BinaryCrossentropy : LossFunctionWrapper, ILossFunc +public class BinaryCrossentropy : LossFunctionWrapper { float label_smoothing; + public BinaryCrossentropy( bool from_logits = false, float label_smoothing = 0, @@ -15,7 +16,6 @@ public class BinaryCrossentropy : LossFunctionWrapper, ILossFunc this.label_smoothing = label_smoothing; } - public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) { var sum = keras.backend.binary_crossentropy(y_true, y_pred, from_logits: from_logits); diff --git a/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs index feb05224..1af57b55 100644 --- a/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs +++ b/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs @@ -1,8 +1,9 @@ namespace Tensorflow.Keras.Losses; -public class CategoricalCrossentropy : LossFunctionWrapper, ILossFunc +public class CategoricalCrossentropy : LossFunctionWrapper { float label_smoothing; + public CategoricalCrossentropy( bool from_logits = false, float label_smoothing = 0, @@ -15,7 +16,6 @@ public class CategoricalCrossentropy : LossFunctionWrapper, ILossFunc this.label_smoothing = label_smoothing; } - public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) { // Try to adjust the shape so that rank of labels = rank of logits - 1. diff --git a/src/TensorFlowNET.Keras/Losses/CosineSimilarity.cs b/src/TensorFlowNET.Keras/Losses/CosineSimilarity.cs index 16ab4b79..cf9df8d0 100644 --- a/src/TensorFlowNET.Keras/Losses/CosineSimilarity.cs +++ b/src/TensorFlowNET.Keras/Losses/CosineSimilarity.cs @@ -1,28 +1,22 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class CosineSimilarity : LossFunctionWrapper { - public class CosineSimilarity : LossFunctionWrapper, ILossFunc + protected int axis = -1; + + public CosineSimilarity( + string reduction = null, + int axis = -1, + string name = null) : + base(reduction: reduction, name: name == null ? "cosine_similarity" : name) { - protected int axis=-1; - public CosineSimilarity( - string reduction = null, - int axis=-1, - string name = null) : - base(reduction: reduction, name: name == null ? "cosine_similarity" : name) - { - this.axis = axis; - } + this.axis = axis; + } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_true_normalize = nn_impl.l2_normalize(y_true, axis : this.axis); - Tensor y_pred_normalize = nn_impl.l2_normalize(y_pred, axis: this.axis); - return -math_ops.reduce_sum(y_true_normalize * y_pred_normalize, axis : constant_op.constant(this.axis)); - } + public override Tensor Apply(Tensor y_true = null, Tensor y_pred = null, bool from_logits = false, int axis = -1) + { + Tensor y_true_normalize = nn_impl.l2_normalize(y_true, axis: this.axis); + Tensor y_pred_normalize = nn_impl.l2_normalize(y_pred, axis: this.axis); + return -math_ops.reduce_sum(y_true_normalize * y_pred_normalize, axis: constant_op.constant(this.axis)); } -} +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Losses/Huber.cs b/src/TensorFlowNET.Keras/Losses/Huber.cs index 7169ba46..61f006d2 100644 --- a/src/TensorFlowNET.Keras/Losses/Huber.cs +++ b/src/TensorFlowNET.Keras/Losses/Huber.cs @@ -1,36 +1,29 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class Huber : LossFunctionWrapper { - public class Huber : LossFunctionWrapper, ILossFunc + protected Tensor delta = tf.Variable(1.0); + + public Huber( + string reduction = null, + Tensor delta = null, + string name = null) : + base(reduction: reduction, name: name == null ? "huber" : name) { - protected Tensor delta = tf.Variable(1.0) ; - public Huber ( - string reduction = null, - Tensor delta = null, - string name = null) : - base(reduction: reduction, name: name == null ? "huber" : name) - { - this.delta = delta==null? this.delta: delta; - - } + this.delta = delta == null ? this.delta : delta; + } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_pred_cast = math_ops.cast(y_pred, dtype: TF_DataType.TF_FLOAT); - Tensor y_true_cast = math_ops.cast(y_true, dtype: TF_DataType.TF_FLOAT); - Tensor delta = math_ops.cast(this.delta, dtype: TF_DataType.TF_FLOAT); - Tensor error = math_ops.subtract(y_pred_cast, y_true_cast); - Tensor abs_error = math_ops.abs(error); - Tensor half = ops.convert_to_tensor(0.5, dtype: abs_error.dtype); - return gen_math_ops.mean(array_ops.where_v2(abs_error <= delta, - half * math_ops.pow(error, 2), - half * math_ops.pow(delta, 2) + delta * (abs_error - delta)), - ops.convert_to_tensor(-1)); - } + public override Tensor Apply(Tensor y_true = null, Tensor y_pred = null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_cast = math_ops.cast(y_pred, dtype: TF_DataType.TF_FLOAT); + Tensor y_true_cast = math_ops.cast(y_true, dtype: TF_DataType.TF_FLOAT); + Tensor delta = math_ops.cast(this.delta, dtype: TF_DataType.TF_FLOAT); + Tensor error = math_ops.subtract(y_pred_cast, y_true_cast); + Tensor abs_error = math_ops.abs(error); + Tensor half = ops.convert_to_tensor(0.5, dtype: abs_error.dtype); + return gen_math_ops.mean(array_ops.where_v2(abs_error <= delta, + half * math_ops.pow(error, 2), + half * math_ops.pow(delta, 2) + delta * (abs_error - delta)), + ops.convert_to_tensor(-1)); } } diff --git a/src/TensorFlowNET.Keras/Losses/LogCosh.cs b/src/TensorFlowNET.Keras/Losses/LogCosh.cs index 7cfd4f67..0c7a9b6e 100644 --- a/src/TensorFlowNET.Keras/Losses/LogCosh.cs +++ b/src/TensorFlowNET.Keras/Losses/LogCosh.cs @@ -1,27 +1,20 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Tensorflow.Operations; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class LogCosh : LossFunctionWrapper { - public class LogCosh : LossFunctionWrapper, ILossFunc - { - public LogCosh( - string reduction = null, - string name = null) : - base(reduction: reduction, name: name == null ? "log_cosh" : name){ } + public LogCosh( + string reduction = null, + string name = null) : + base(reduction: reduction, name: name == null ? "log_cosh" : name) + { } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); - Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); - Tensor x = y_pred_dispatch - y_true_cast; + public override Tensor Apply(Tensor y_true = null, Tensor y_pred = null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); + Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); + Tensor x = y_pred_dispatch - y_true_cast; - return gen_math_ops.mean(x + gen_nn_ops.softplus(-2.0 * x) - math_ops.cast(math_ops.log(tf.Variable(2.0)), x.dtype), - ops.convert_to_tensor(-1)); - } + return gen_math_ops.mean(x + gen_nn_ops.softplus(-2.0 * x) - math_ops.cast(math_ops.log(tf.Variable(2.0)), x.dtype), + ops.convert_to_tensor(-1)); } -} +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Losses/Loss.cs b/src/TensorFlowNET.Keras/Losses/Loss.cs index 77bf7e1d..ce77f6d6 100644 --- a/src/TensorFlowNET.Keras/Losses/Loss.cs +++ b/src/TensorFlowNET.Keras/Losses/Loss.cs @@ -1,55 +1,51 @@ -using System; -using Tensorflow.Keras.Utils; +using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Losses +namespace Tensorflow.Keras.Losses; + +/// +/// Loss base class. +/// +public abstract class Loss : ILossFunc { - /// - /// Loss base class. - /// - public abstract class Loss + protected string reduction; + protected string name; + bool _allow_sum_over_batch_size; + protected bool from_logits = false; + string _name_scope; + + public string Reduction => reduction; + public string Name => name; + + public Loss(string reduction = ReductionV2.AUTO, + string name = null, + bool from_logits = false) { - protected string reduction; - protected string name; - bool _allow_sum_over_batch_size; - protected bool from_logits = false; - string _name_scope; - - public string Reduction => reduction; - public string Name => name; - public Loss(string reduction = ReductionV2.AUTO, - string name = null, - bool from_logits = false) - { - this.reduction = reduction == null ? ReductionV2.SUM_OVER_BATCH_SIZE : reduction; - this.name = name; - this.from_logits = from_logits; - _allow_sum_over_batch_size = false; - } + this.reduction = reduction == null ? ReductionV2.SUM_OVER_BATCH_SIZE : reduction; + this.name = name; + this.from_logits = from_logits; + _allow_sum_over_batch_size = false; + } - public virtual Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) - { - throw new NotImplementedException(""); - } + public abstract Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1); - public Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) - { - var losses = Apply(y_true, y_pred, from_logits: from_logits); - var reduction = GetReduction(); - return losses_utils.compute_weighted_loss(losses, reduction: reduction, sample_weight: sample_weight); - } + public Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) + { + var losses = Apply(y_true, y_pred, from_logits: from_logits); + var reduction = GetReduction(); + return losses_utils.compute_weighted_loss(losses, reduction: reduction, sample_weight: sample_weight); + } - string GetReduction() - { - return reduction switch - { - ReductionV2.AUTO => ReductionV2.SUM_OVER_BATCH_SIZE, - _ => reduction - }; - } - - void _set_name_scope() + string GetReduction() + { + return reduction switch { - _name_scope = name; - } + ReductionV2.AUTO => ReductionV2.SUM_OVER_BATCH_SIZE, + _ => reduction + }; + } + + void _set_name_scope() + { + _name_scope = name; } -} +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Losses/LossFunctionWrapper.cs b/src/TensorFlowNET.Keras/Losses/LossFunctionWrapper.cs index 758b46f4..f4ee2b34 100644 --- a/src/TensorFlowNET.Keras/Losses/LossFunctionWrapper.cs +++ b/src/TensorFlowNET.Keras/Losses/LossFunctionWrapper.cs @@ -1,16 +1,14 @@ using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Losses +namespace Tensorflow.Keras.Losses; + +public abstract class LossFunctionWrapper : Loss { - public class LossFunctionWrapper : Loss - { - public LossFunctionWrapper(string reduction = ReductionV2.AUTO, - string name = null, - bool from_logits = false) - : base(reduction: reduction, - name: name, - from_logits: from_logits) - { - } - } + public LossFunctionWrapper(string reduction = ReductionV2.AUTO, + string name = null, + bool from_logits = false) + : base(reduction: reduction, + name: name, + from_logits: from_logits) + { } } diff --git a/src/TensorFlowNET.Keras/Losses/MeanAbsoluteError.cs b/src/TensorFlowNET.Keras/Losses/MeanAbsoluteError.cs index c203bc5a..19476a68 100644 --- a/src/TensorFlowNET.Keras/Losses/MeanAbsoluteError.cs +++ b/src/TensorFlowNET.Keras/Losses/MeanAbsoluteError.cs @@ -1,23 +1,16 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class MeanAbsoluteError : LossFunctionWrapper { - public class MeanAbsoluteError : LossFunctionWrapper, ILossFunc - { - public MeanAbsoluteError( - string reduction = null, - string name = null) : - base(reduction: reduction, name: name == null ? "mean_absolute_error" : name){ } + public MeanAbsoluteError( + string reduction = null, + string name = null) : + base(reduction: reduction, name: name == null ? "mean_absolute_error" : name){ } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); - Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); - return gen_math_ops.mean(math_ops.abs(y_pred_dispatch - y_true_cast), ops.convert_to_tensor(-1)); - } + public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); + Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); + return gen_math_ops.mean(math_ops.abs(y_pred_dispatch - y_true_cast), ops.convert_to_tensor(-1)); } } diff --git a/src/TensorFlowNET.Keras/Losses/MeanAbsolutePercentageError.cs b/src/TensorFlowNET.Keras/Losses/MeanAbsolutePercentageError.cs index 8dcaa1bc..226c4237 100644 --- a/src/TensorFlowNET.Keras/Losses/MeanAbsolutePercentageError.cs +++ b/src/TensorFlowNET.Keras/Losses/MeanAbsolutePercentageError.cs @@ -1,24 +1,17 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class MeanAbsolutePercentageError : LossFunctionWrapper { - public class MeanAbsolutePercentageError : LossFunctionWrapper, ILossFunc - { - public MeanAbsolutePercentageError( - string reduction = null, - string name = null) : - base(reduction: reduction, name: name == null ? "mean_absolute_percentage_error" : name){ } + public MeanAbsolutePercentageError( + string reduction = null, + string name = null) : + base(reduction: reduction, name: name == null ? "mean_absolute_percentage_error" : name){ } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); - Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); - Tensor diff = math_ops.abs(y_true_cast - y_pred_dispatch) / gen_math_ops.maximum(math_ops.abs(y_true_cast), gen_math_ops.cast(tf.constant(1e-7), y_pred_dispatch.dtype)); - return gen_math_ops.cast(tf.constant(100), y_pred_dispatch.dtype) * gen_math_ops.mean(diff, ops.convert_to_tensor(-1)); - } + public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); + Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); + Tensor diff = math_ops.abs(y_true_cast - y_pred_dispatch) / gen_math_ops.maximum(math_ops.abs(y_true_cast), gen_math_ops.cast(tf.constant(1e-7), y_pred_dispatch.dtype)); + return gen_math_ops.cast(tf.constant(100), y_pred_dispatch.dtype) * gen_math_ops.mean(diff, ops.convert_to_tensor(-1)); } } diff --git a/src/TensorFlowNET.Keras/Losses/MeanSquaredError.cs b/src/TensorFlowNET.Keras/Losses/MeanSquaredError.cs index 73cddef1..a937c196 100644 --- a/src/TensorFlowNET.Keras/Losses/MeanSquaredError.cs +++ b/src/TensorFlowNET.Keras/Losses/MeanSquaredError.cs @@ -1,23 +1,16 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class MeanSquaredError : LossFunctionWrapper { - public class MeanSquaredError : LossFunctionWrapper, ILossFunc - { - public MeanSquaredError( - string reduction = null, - string name = null) : - base(reduction: reduction, name: name==null? "mean_squared_error" : name){ } + public MeanSquaredError( + string reduction = null, + string name = null) : + base(reduction: reduction, name: name==null? "mean_squared_error" : name){ } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) - { - Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); - Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); - return gen_math_ops.mean(gen_math_ops.squared_difference(y_pred_dispatch, y_true_cast), ops.convert_to_tensor(-1)); - } + public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); + Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); + return gen_math_ops.mean(gen_math_ops.squared_difference(y_pred_dispatch, y_true_cast), ops.convert_to_tensor(-1)); } } diff --git a/src/TensorFlowNET.Keras/Losses/MeanSquaredLogarithmicError.cs b/src/TensorFlowNET.Keras/Losses/MeanSquaredLogarithmicError.cs index e2965921..0a4e7d3c 100644 --- a/src/TensorFlowNET.Keras/Losses/MeanSquaredLogarithmicError.cs +++ b/src/TensorFlowNET.Keras/Losses/MeanSquaredLogarithmicError.cs @@ -1,33 +1,28 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class MeanSquaredLogarithmicError : LossFunctionWrapper { - public class MeanSquaredLogarithmicError : LossFunctionWrapper, ILossFunc - { - public MeanSquaredLogarithmicError( - string reduction = null, - string name = null) : - base(reduction: reduction, name: name == null ? "mean_squared_logarithmic_error" : name){ } - + public MeanSquaredLogarithmicError( + string reduction = null, + string name = null) : + base(reduction: reduction, name: name == null ? "mean_squared_logarithmic_error" : name) + { } - public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1) + public override Tensor Apply(Tensor y_true = null, Tensor y_pred = null, bool from_logits = false, int axis = -1) + { + Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); + Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); + Tensor first_log = null, second_log = null; + if (y_pred_dispatch.dtype == TF_DataType.TF_DOUBLE) + { + first_log = math_ops.log(math_ops.maximum(y_pred_dispatch, 1e-7) + 1.0); + second_log = math_ops.log(math_ops.maximum(y_true_cast, 1e-7) + 1.0); + } + else { - Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred); - Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype); - Tensor first_log=null, second_log=null; - if (y_pred_dispatch.dtype == TF_DataType.TF_DOUBLE) { - first_log = math_ops.log(math_ops.maximum(y_pred_dispatch, 1e-7) + 1.0); - second_log = math_ops.log(math_ops.maximum(y_true_cast, 1e-7) + 1.0); - } - else { - first_log = math_ops.log(math_ops.maximum(y_pred_dispatch, 1e-7f) + 1.0f); - second_log = math_ops.log(math_ops.maximum(y_true_cast, 1e-7f) + 1.0f); - } - return gen_math_ops.mean(gen_math_ops.squared_difference(first_log, second_log), ops.convert_to_tensor(-1)); + first_log = math_ops.log(math_ops.maximum(y_pred_dispatch, 1e-7f) + 1.0f); + second_log = math_ops.log(math_ops.maximum(y_true_cast, 1e-7f) + 1.0f); } + return gen_math_ops.mean(gen_math_ops.squared_difference(first_log, second_log), ops.convert_to_tensor(-1)); } -} +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs b/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs index 7ac3fa0b..ec6dcedf 100644 --- a/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs +++ b/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs @@ -2,7 +2,7 @@ namespace Tensorflow.Keras.Losses; -public class SigmoidFocalCrossEntropy : LossFunctionWrapper, ILossFunc +public class SigmoidFocalCrossEntropy : LossFunctionWrapper { float _alpha; float _gamma; @@ -20,7 +20,6 @@ public class SigmoidFocalCrossEntropy : LossFunctionWrapper, ILossFunc _gamma = gamma; } - public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) { y_true = tf.cast(y_true, dtype: y_pred.dtype); diff --git a/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs index 4e2790ab..17ce2d30 100644 --- a/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs +++ b/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs @@ -1,41 +1,41 @@ using static Tensorflow.Binding; -namespace Tensorflow.Keras.Losses +namespace Tensorflow.Keras.Losses; + +public class SparseCategoricalCrossentropy : LossFunctionWrapper { - public class SparseCategoricalCrossentropy : LossFunctionWrapper, ILossFunc + private bool _from_logits = false; + + public SparseCategoricalCrossentropy( + bool from_logits = false, + string reduction = null, + string name = null) : + base(reduction: reduction, name: name == null ? "sparse_categorical_crossentropy" : name) + { + _from_logits = from_logits; + } + + public override Tensor Apply(Tensor target, Tensor output, bool from_logits = false, int axis = -1) { - private bool _from_logits = false; - public SparseCategoricalCrossentropy( - bool from_logits = false, - string reduction = null, - string name = null) : - base(reduction: reduction, name: name == null ? "sparse_categorical_crossentropy" : name) + target = tf.cast(target, dtype: TF_DataType.TF_INT64); + + if (!_from_logits) { - _from_logits = from_logits; + var epsilon = tf.constant(KerasApi.keras.backend.epsilon(), output.dtype); + output = tf.clip_by_value(output, epsilon, 1 - epsilon); + output = tf.log(output); } - public override Tensor Apply(Tensor target, Tensor output, bool from_logits = false, int axis = -1) + // Try to adjust the shape so that rank of labels = rank of logits - 1. + var output_shape = array_ops.shape_v2(output); + var output_rank = output.shape.ndim; + var target_rank = target.shape.ndim; + var update_shape = target_rank != output_rank - 1; + if (update_shape) { - target = tf.cast(target, dtype: TF_DataType.TF_INT64); - - if (!_from_logits) - { - var epsilon = tf.constant(KerasApi.keras.backend.epsilon(), output.dtype); - output = tf.clip_by_value(output, epsilon, 1 - epsilon); - output = tf.log(output); - } - - // Try to adjust the shape so that rank of labels = rank of logits - 1. - var output_shape = array_ops.shape_v2(output); - var output_rank = output.shape.ndim; - var target_rank = target.shape.ndim; - var update_shape = target_rank != output_rank - 1; - if (update_shape) - { - target = array_ops.reshape(target, new int[] { -1 }); - output = array_ops.reshape(output, new int[] { -1, output_shape[-1].numpy() }); - } - return tf.nn.sparse_softmax_cross_entropy_with_logits(target, output); + target = array_ops.reshape(target, new int[] { -1 }); + output = array_ops.reshape(output, new int[] { -1, output_shape[-1].numpy() }); } + return tf.nn.sparse_softmax_cross_entropy_with_logits(target, output); } -} +} \ No newline at end of file From 46e190dbfc871ce4dd780d58d888d6406cc0285e Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Tue, 6 Jun 2023 11:12:49 +0800 Subject: [PATCH 02/77] feat: add RNN basic framework. --- .../Extensions/DictionaryExtension.cs | 0 .../Extensions/JObjectExtensions.cs | 6 +- .../Common/Extensions/LinqExtensions.cs | 26 + .../{ => Common}/Extensions/OneofExtension.cs | 0 .../Common/Types/GeneralizedTensorShape.cs | 79 +++ .../Common/Types/IOptionalArgs.cs | 21 + .../Types}/NamedTuple.cs | 0 .../Types}/TensorShapeConfig.cs | 2 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 11 +- .../ArgsDefinition/Rnn/RnnOptionalArgs.cs | 14 + .../ArgsDefinition/Rnn/SimpleRNNCellArgs.cs | 29 + src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 5 +- .../Keras/Layers/Rnn/IRnnCell.cs | 19 + .../Keras/Layers/Rnn/IStackedRnnCells.cs | 12 + ...stomizedKerasShapesWrapperJsonConverter.cs | 1 + .../Keras/Saving/KerasShapesWrapper.cs | 1 + src/TensorFlowNET.Core/NumPy/Axis.cs | 5 - .../Operations/Initializers/Orthogonal.cs | 2 +- .../Operations/NnOps/BasicLSTMCell.cs | 1 + .../Operations/NnOps/BasicRNNCell.cs | 1 + .../Operations/NnOps/LayerRNNCell.cs | 1 + .../Operations/NnOps/RNNCell.cs | 15 +- .../Operations/logging_ops.cs | 2 +- src/TensorFlowNET.Core/Operations/sort_ops.cs | 2 +- .../Tensorflow.Binding.csproj | 5 + src/TensorFlowNET.Core/Tensors/Tensors.cs | 40 +- src/TensorFlowNET.Core/Util/nest.py.cs | 33 + src/TensorFlowNET.Keras/BackendImpl.cs | 510 ++++++++++++++++ src/TensorFlowNET.Keras/Engine/Functional.cs | 5 +- src/TensorFlowNET.Keras/Engine/Layer.Apply.cs | 7 +- src/TensorFlowNET.Keras/Engine/Layer.cs | 4 +- src/TensorFlowNET.Keras/Engine/Model.cs | 2 +- src/TensorFlowNET.Keras/Engine/Sequential.cs | 3 +- .../Layers/Activation/ELU.cs | 3 +- .../Layers/Activation/Exponential.cs | 4 +- .../Layers/Activation/HardSigmoid.cs | 3 +- .../Layers/Activation/LeakyReLu.cs | 3 +- .../Layers/Activation/SELU.cs | 3 +- .../Layers/Activation/Softmax.cs | 5 +- .../Layers/Activation/Softplus.cs | 3 +- .../Layers/Activation/Softsign.cs | 3 +- .../Layers/Activation/Swish.cs | 3 +- .../Layers/Activation/Tanh.cs | 3 +- .../Layers/Attention/BaseDenseAttention.cs | 3 +- .../Layers/Attention/MultiHeadAttention.cs | 5 +- .../Layers/Convolution/Conv2DTranspose.cs | 3 +- .../Layers/Convolution/Convolutional.cs | 3 +- src/TensorFlowNET.Keras/Layers/Core/Dense.cs | 3 +- .../Layers/Core/EinsumDense.cs | 3 +- .../Layers/Core/Embedding.cs | 3 +- .../Layers/Merging/Merge.cs | 3 +- .../Normalization/BatchNormalization.cs | 3 +- .../Normalization/LayerNormalization.cs | 3 +- .../Layers/Normalization/Normalization.cs | 3 +- .../Layers/Pooling/GlobalAveragePooling1D.cs | 3 +- .../Layers/Pooling/GlobalAveragePooling2D.cs | 3 +- .../Layers/Pooling/GlobalMaxPooling1D.cs | 3 +- .../Layers/Pooling/GlobalMaxPooling2D.cs | 3 +- .../Layers/Pooling/Pooling1D.cs | 3 +- .../Layers/Pooling/Pooling2D.cs | 3 +- .../Layers/Preprocessing/CategoryEncoding.cs | 4 +- .../Layers/Preprocessing/Rescaling.cs | 3 +- .../Layers/Preprocessing/Resizing.cs | 3 +- .../Layers/Regularization/Dropout.cs | 5 +- .../Layers/Reshaping/Cropping1D.cs | 4 +- .../Layers/Reshaping/Cropping2D.cs | 3 +- .../Layers/Reshaping/Cropping3D.cs | 3 +- .../Layers/Reshaping/Flatten.cs | 3 +- .../Layers/Reshaping/Permute.cs | 3 +- .../Layers/Reshaping/Reshape.cs | 3 +- .../Layers/Reshaping/UpSampling2D.cs | 3 +- .../Layers/Reshaping/ZeroPadding2D.cs | 3 +- .../Layers/Rnn/DropoutRNNCellMixin.cs | 85 +++ src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 5 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 569 +++++++++++++++--- src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs | 13 + .../Layers/Rnn/RnnCellBase.cs | 24 + .../Layers/Rnn/SimpleRNN.cs | 22 +- .../Layers/Rnn/SimpleRNNCell.cs | 113 +++- .../Layers/Rnn/StackedRNNCells.cs | 13 +- .../Layers/TensorFlowOpLayer.cs | 3 +- .../Metrics/metrics_utils.cs | 2 +- ...processing.image_dataset_from_directory.cs | 2 +- .../Saving/KerasObjectLoader.cs | 2 +- src/TensorFlowNET.Keras/Utils/RnnUtils.cs | 93 +++ .../Layers/LayersTest.cs | 11 - .../Layers/Rnn.Test.cs | 28 + tools/TensorFlowNET.Console/SimpleRnnTest.cs | 2 +- 88 files changed, 1789 insertions(+), 188 deletions(-) rename src/TensorFlowNET.Core/{ => Common}/Extensions/DictionaryExtension.cs (100%) rename src/TensorFlowNET.Core/{ => Common}/Extensions/JObjectExtensions.cs (80%) create mode 100644 src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs rename src/TensorFlowNET.Core/{ => Common}/Extensions/OneofExtension.cs (100%) create mode 100644 src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/IOptionalArgs.cs rename src/TensorFlowNET.Core/{Extensions => Common/Types}/NamedTuple.cs (100%) rename src/TensorFlowNET.Core/{Keras/Saving => Common/Types}/TensorShapeConfig.cs (95%) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs create mode 100644 src/TensorFlowNET.Keras/Utils/RnnUtils.cs create mode 100644 test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs diff --git a/src/TensorFlowNET.Core/Extensions/DictionaryExtension.cs b/src/TensorFlowNET.Core/Common/Extensions/DictionaryExtension.cs similarity index 100% rename from src/TensorFlowNET.Core/Extensions/DictionaryExtension.cs rename to src/TensorFlowNET.Core/Common/Extensions/DictionaryExtension.cs diff --git a/src/TensorFlowNET.Core/Extensions/JObjectExtensions.cs b/src/TensorFlowNET.Core/Common/Extensions/JObjectExtensions.cs similarity index 80% rename from src/TensorFlowNET.Core/Extensions/JObjectExtensions.cs rename to src/TensorFlowNET.Core/Common/Extensions/JObjectExtensions.cs index 2e758dbf..6ceba445 100644 --- a/src/TensorFlowNET.Core/Extensions/JObjectExtensions.cs +++ b/src/TensorFlowNET.Core/Common/Extensions/JObjectExtensions.cs @@ -3,16 +3,16 @@ using System; using System.Collections.Generic; using System.Text; -namespace Tensorflow.Extensions +namespace Tensorflow.Common.Extensions { public static class JObjectExtensions { public static T? TryGetOrReturnNull(this JObject obj, string key) { var res = obj[key]; - if(res is null) + if (res is null) { - return default(T); + return default; } else { diff --git a/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs new file mode 100644 index 00000000..0402fca0 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tensorflow.Common.Extensions +{ + public static class LinqExtensions + { +#if NETSTANDARD2_0 + public static IEnumerable TakeLast(this IEnumerable sequence, int count) + { + return sequence.Skip(sequence.Count() - count); + } + + public static IEnumerable SkipLast(this IEnumerable sequence, int count) + { + return sequence.Take(sequence.Count() - count); + } +#endif + public static Tensors ToTensors(this IEnumerable tensors) + { + return new Tensors(tensors); + } + } +} diff --git a/src/TensorFlowNET.Core/Extensions/OneofExtension.cs b/src/TensorFlowNET.Core/Common/Extensions/OneofExtension.cs similarity index 100% rename from src/TensorFlowNET.Core/Extensions/OneofExtension.cs rename to src/TensorFlowNET.Core/Common/Extensions/OneofExtension.cs diff --git a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs new file mode 100644 index 00000000..edb9a802 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; + +namespace Tensorflow.Common.Types +{ + public class GeneralizedTensorShape: IEnumerable + { + public TensorShapeConfig[] Shapes { get; set; } + /// + /// create a single-dim generalized Tensor shape. + /// + /// + public GeneralizedTensorShape(int dim) + { + Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; + } + + public GeneralizedTensorShape(Shape shape) + { + Shapes = new TensorShapeConfig[] { shape }; + } + + public GeneralizedTensorShape(TensorShapeConfig shape) + { + Shapes = new TensorShapeConfig[] { shape }; + } + + public GeneralizedTensorShape(TensorShapeConfig[] shapes) + { + Shapes = shapes; + } + + public GeneralizedTensorShape(IEnumerable shape) + { + Shapes = shape.Select(x => (TensorShapeConfig)x).ToArray(); + } + + public Shape ToSingleShape() + { + if (Shapes.Length != 1) + { + throw new ValueError("The generalized shape contains more than 1 dim."); + } + var shape_config = Shapes[0]; + Debug.Assert(shape_config is not null); + return new Shape(shape_config.Items.Select(x => x is null ? -1 : x.Value).ToArray()); + } + + public long ToNumber() + { + if(Shapes.Length != 1 || Shapes[0].Items.Length != 1) + { + throw new ValueError("The generalized shape contains more than 1 dim."); + } + var res = Shapes[0].Items[0]; + return res is null ? -1 : res.Value; + } + + public Shape[] ToShapeArray() + { + return Shapes.Select(x => new Shape(x.Items.Select(y => y is null ? -1 : y.Value).ToArray())).ToArray(); + } + + public IEnumerator GetEnumerator() + { + foreach (var shape in Shapes) + { + yield return shape.Items; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/IOptionalArgs.cs b/src/TensorFlowNET.Core/Common/Types/IOptionalArgs.cs new file mode 100644 index 00000000..427e71aa --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/IOptionalArgs.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + /// + /// This interface is used when some corresponding python methods have optional args. + /// For example, `Keras.Layer.Apply` generally takes three args as the inputs, while + /// `Keras.Layer.RNN` takes more. Then when calling RNN, you should add `RnnOptionalArgs` + /// as the parameter of the method. + /// + public interface IOptionalArgs + { + /// + /// The identifier of the class. It is not an argument but only something to + /// separate different OptionalArgs. + /// + string Identifier { get; } + } +} diff --git a/src/TensorFlowNET.Core/Extensions/NamedTuple.cs b/src/TensorFlowNET.Core/Common/Types/NamedTuple.cs similarity index 100% rename from src/TensorFlowNET.Core/Extensions/NamedTuple.cs rename to src/TensorFlowNET.Core/Common/Types/NamedTuple.cs diff --git a/src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs b/src/TensorFlowNET.Core/Common/Types/TensorShapeConfig.cs similarity index 95% rename from src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs rename to src/TensorFlowNET.Core/Common/Types/TensorShapeConfig.cs index 7abcfde2..a36930ec 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs +++ b/src/TensorFlowNET.Core/Common/Types/TensorShapeConfig.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; -namespace Tensorflow.Keras.Saving +namespace Tensorflow.Common.Types { public class TensorShapeConfig { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 2585592c..ed5a1d6d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -1,17 +1,15 @@ using Newtonsoft.Json; using System.Collections.Generic; +using Tensorflow.Keras.Layers.Rnn; namespace Tensorflow.Keras.ArgsDefinition.Rnn { + // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs { - public interface IRnnArgCell : ILayer - { - object state_size { get; } - } [JsonProperty("cell")] // TODO: the cell should be serialized with `serialize_keras_object`. - public IRnnArgCell Cell { get; set; } = null; + public IRnnCell Cell { get; set; } = null; [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; [JsonProperty("return_state")] @@ -34,6 +32,9 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public IInitializer KernelInitializer { get; set; } public IInitializer RecurrentInitializer { get; set; } public IInitializer BiasInitializer { get; set; } + public float Dropout { get; set; } = .0f; + public bool ZeroOutputForMask { get; set; } = false; + public float RecurrentDropout { get; set; } = .0f; // kernel_regularizer=None, // recurrent_regularizer=None, diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs new file mode 100644 index 00000000..64b500bb --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class RnnOptionalArgs: IOptionalArgs + { + public string Identifier => "Rnn"; + public Tensor Mask { get; set; } = null; + public Tensors Constants { get; set; } = null; + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs new file mode 100644 index 00000000..1dfcbe9c --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs @@ -0,0 +1,29 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class SimpleRNNCellArgs: AutoSerializeLayerArgs + { + [JsonProperty("units")] + public int Units { get; set; } + // TODO(Rinne): lack of initialized value of Activation. Merging keras + // into tf.net could resolve it. + [JsonProperty("activation")] + public Activation Activation { get; set; } + [JsonProperty("use_bias")] + public bool UseBias { get; set; } = true; + [JsonProperty("dropout")] + public float Dropout { get; set; } = .0f; + [JsonProperty("recurrent_dropout")] + public float RecurrentDropout { get; set; } = .0f; + [JsonProperty("kernel_initializer")] + public IInitializer KernelInitializer { get; set; } + [JsonProperty("recurrent_initializer")] + public IInitializer RecurrentInitializer { get; set; } + [JsonProperty("bias_initializer")] + public IInitializer BiasInitializer { get; set; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index f7669394..e94c8bf1 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -1,4 +1,5 @@ -using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Training; @@ -14,7 +15,7 @@ namespace Tensorflow.Keras List Layers { get; } List InboundNodes { get; } List OutboundNodes { get; } - Tensors Apply(Tensors inputs, Tensor state = null, bool training = false); + Tensors Apply(Tensors inputs, Tensors states = null, bool training = false, IOptionalArgs? optional_args = null); List TrainableVariables { get; } List TrainableWeights { get; } List NonTrainableWeights { get; } diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs new file mode 100644 index 00000000..df6222cd --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public interface IRnnCell: ILayer + { + GeneralizedTensorShape StateSize { get; } + GeneralizedTensorShape OutputSize { get; } + /// + /// Whether the optional RNN args are supported when appying the layer. + /// In other words, whether `Apply` is overwrited with process of `RnnOptionalArgs`. + /// + bool SupportOptionalArgs { get; } + (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs new file mode 100644 index 00000000..e73244a5 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public interface IStackedRnnCells : IRnnCell + { + int Count { get; } + IRnnCell this[int idx] { get; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedKerasShapesWrapperJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedKerasShapesWrapperJsonConverter.cs index 1a4245bf..3a21db9d 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedKerasShapesWrapperJsonConverter.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/Json/CustomizedKerasShapesWrapperJsonConverter.cs @@ -3,6 +3,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Saving.Json { diff --git a/src/TensorFlowNET.Core/Keras/Saving/KerasShapesWrapper.cs b/src/TensorFlowNET.Core/Keras/Saving/KerasShapesWrapper.cs index d91d3161..ea6fe976 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/KerasShapesWrapper.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/KerasShapesWrapper.cs @@ -6,6 +6,7 @@ using System.Text; using System.Diagnostics; using OneOf.Types; using Tensorflow.Keras.Saving.Json; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Saving { diff --git a/src/TensorFlowNET.Core/NumPy/Axis.cs b/src/TensorFlowNET.Core/NumPy/Axis.cs index 976c764f..7a3ecbf1 100644 --- a/src/TensorFlowNET.Core/NumPy/Axis.cs +++ b/src/TensorFlowNET.Core/NumPy/Axis.cs @@ -74,8 +74,3 @@ namespace Tensorflow => IsScalar ? $"{axis[0]}" : $"({string.Join(", ", axis)})"; } } - -namespace System.Runtime.CompilerServices -{ - internal static class IsExternalInit { } -} diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs index 492047c9..88673bb5 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs @@ -53,7 +53,7 @@ public class Orthogonal : IInitializer // Compute the qr factorization var (q, r) = tf.linalg.qr(a, full_matrices: false); // Make Q uniform - var d = tf.linalg.tensor_diag_part(r); + var d = tf.linalg.tensor_diag_part(r.Single); q *= tf.sign(d); if (num_rows < num_cols) diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs index d3592514..b2cda952 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs @@ -11,6 +11,7 @@ namespace Tensorflow /// Basic LSTM recurrent network cell. /// The implementation is based on: http://arxiv.org/abs/1409.2329. /// + [Obsolete("This is an incompleted tf v1 api, pleas use keras RNNs instead.")] public class BasicLstmCell : LayerRnnCell { int _num_units; diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs index 17d51363..3308aebb 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/BasicRNNCell.cs @@ -20,6 +20,7 @@ using static Tensorflow.Binding; namespace Tensorflow { + [Obsolete("This is an incompleted tf v1 api, pleas use keras RNNs instead.")] public class BasicRnnCell : LayerRnnCell { int _num_units; diff --git a/src/TensorFlowNET.Core/Operations/NnOps/LayerRNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/LayerRNNCell.cs index 7394cb7f..65de4fe9 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/LayerRNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/LayerRNNCell.cs @@ -19,6 +19,7 @@ using static Tensorflow.Binding; namespace Tensorflow { + [Obsolete("This is an incompleted tf v1 api, pleas use keras RNNs instead.")] public class LayerRnnCell : RnnCell { protected InputSpec inputSpec; diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index ecc9ca11..71fdc301 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -16,10 +16,12 @@ using System; using System.Collections.Generic; +using Tensorflow.Common.Types; using Tensorflow.Keras; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Rnn; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Operations; @@ -50,7 +52,8 @@ namespace Tensorflow /// matching structure of Tensors having shape `[batch_size].concatenate(s)` /// for each `s` in `self.batch_size`. /// - public abstract class RnnCell : ILayer, RNNArgs.IRnnArgCell + [Obsolete("This is an incompleted tf v1 api, pleas use keras RNNs instead.")] + public abstract class RnnCell : ILayer, IRnnCell { /// /// Attribute that indicates whether the cell is a TF RNN cell, due the slight @@ -142,7 +145,7 @@ namespace Tensorflow throw new NotImplementedException("_zero_state_tensors"); } - public Tensors Apply(Tensors inputs, Tensor state = null, bool is_training = false) + public Tensors Apply(Tensors inputs, Tensors state = null, bool is_training = false, IOptionalArgs? optional_args = null) { throw new NotImplementedException(); } @@ -173,5 +176,13 @@ namespace Tensorflow { throw new NotImplementedException(); } + + public (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) + { + throw new NotImplementedException(); + } + public GeneralizedTensorShape StateSize => throw new NotImplementedException(); + public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); + public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/src/TensorFlowNET.Core/Operations/logging_ops.cs b/src/TensorFlowNET.Core/Operations/logging_ops.cs index e38e60b5..3303cadc 100644 --- a/src/TensorFlowNET.Core/Operations/logging_ops.cs +++ b/src/TensorFlowNET.Core/Operations/logging_ops.cs @@ -30,7 +30,7 @@ namespace Tensorflow name: name); return tf.Context.ExecuteOp("PrintV2", name, new ExecuteOpArgs(formatted_string) - .SetAttributes(new { output_stream, end })); + .SetAttributes(new { output_stream, end })).SingleOrNull; } } } diff --git a/src/TensorFlowNET.Core/Operations/sort_ops.cs b/src/TensorFlowNET.Core/Operations/sort_ops.cs index 34b90323..db38a073 100644 --- a/src/TensorFlowNET.Core/Operations/sort_ops.cs +++ b/src/TensorFlowNET.Core/Operations/sort_ops.cs @@ -44,7 +44,7 @@ namespace Tensorflow { sorted = true })); - return indices; + return indices.Single; } public static Tensor sort(Tensor values, Axis axis, string direction = "ASCENDING", string? name = null) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 09f5b077..b08b2e2b 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -114,4 +114,9 @@ https://tensorflownet.readthedocs.io + + + + + diff --git a/src/TensorFlowNET.Core/Tensors/Tensors.cs b/src/TensorFlowNET.Core/Tensors/Tensors.cs index d063ee39..caa36b76 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensors.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensors.cs @@ -23,6 +23,38 @@ namespace Tensorflow public Graph graph => items.First().graph; public bool IsList { get; set; } public int Length => items.Count(); + /// + /// Return a Tensor if `Tensors` has only one tensor, otherwise throw an exception. + /// + public Tensor Single + { + get + { + if (Length != 1) + { + throw new ValueError("Tensors with more than one tensor cannot be " + + "implicitly converted to Tensor."); + } + return items.First(); + } + } + + /// + /// Return a Tensor if `Tensors` has only one tensor, and return null when `Tensors` is empty, + /// otherwise throw an exception. + /// + public Tensor? SingleOrNull + { + get + { + if (Length > 1) + { + throw new ValueError($"Tensors with {Length} tensor cannot be " + + "implicitly converted to Tensor."); + } + return items.FirstOrDefault(); + } + } public Tensor this[int index] { @@ -183,18 +215,18 @@ namespace Tensorflow public static implicit operator Tensors(List tensors) => new Tensors(tensors.ToArray()); - public static implicit operator Tensor(Tensors tensors) - => tensors.FirstOrDefault(); + public static implicit operator Tensor(Tensors? tensors) + => tensors?.SingleOrNull; public static implicit operator Tensor[](Tensors tensors) => tensors.items.ToArray(); #endregion - public void Deconstruct(out Tensor a, out Tensor b) + public void Deconstruct(out Tensor a, out Tensors? b) { a = items[0]; - b = items[1]; + b = Length == 1? null : new Tensors(items.Skip(1)); } private static void EnsureSingleTensor(Tensors tensors, string methodnName) diff --git a/src/TensorFlowNET.Core/Util/nest.py.cs b/src/TensorFlowNET.Core/Util/nest.py.cs index eb94f4d0..ab6f56b3 100644 --- a/src/TensorFlowNET.Core/Util/nest.py.cs +++ b/src/TensorFlowNET.Core/Util/nest.py.cs @@ -170,6 +170,39 @@ namespace Tensorflow.Util throw new TypeError("Type of sequence not supported (yet): " + instance.GetType()); } + public static bool is_nested(object obj) + { + // Refer to https://www.tensorflow.org/api_docs/python/tf/nest + //if (obj is IList || obj is IDictionary || obj is ITuple) + // return true; + if (obj is IList || obj is IDictionary) + return true; + + if (obj is NDArray || obj is Tensor || obj is string || obj.GetType().IsGenericType + || obj is ISet || obj is ISet || obj is ISet) + return false; + + if (obj.GetType().IsNested) return true; + // Check if the object is an IEnumerable + if (obj is IEnumerable) + { + // If it is, check if it is a nested structure + foreach (object item in (IEnumerable)obj) + { + if (is_nested(item)) + { + return true; + } + } + return true; + } + else + { + // If it is not, return false + return false; + } + } + /// /// Yields the next value from the given iterable. /// diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 80403ad6..a7c1bcad 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -22,6 +22,7 @@ using Tensorflow.Functions; using Tensorflow.Graphs; using static Tensorflow.Binding; using static Tensorflow.Graphs.SubGraphUtility; +using Tensorflow.Util; namespace Tensorflow.Keras { @@ -450,5 +451,514 @@ namespace Tensorflow.Keras return x; } + + public static (Tensors, Tensors, Tensors) rnn( + Func step_function, // args:inputs, states, return:output, new_states + Tensors inputs, // inputs is a tuple of tensors (one per input sequence) + Tensors initial_states, + bool go_backwards = false, + Tensor? mask = null, + Tensors? constants = null, + bool unroll = false, + Tensors? input_length = null, // An integer or a 1-D Tensor,depending on whether the time dimension is fixed-length or not + bool time_major = false, + bool zero_output_for_mask = false, + bool return_all_outputs = true) + { + + Tensors swap_batch_timestep(Tensors input_t) + { + var axes = Enumerable.Range(0, input_t.rank).ToArray(); + axes[0] = 1; + axes[1] = 0; + return tf.transpose(input_t, axes); + } + + if (!time_major) + { + inputs = nest.map_structure(swap_batch_timestep, inputs); + } + + var flatted_inptus = nest.flatten(inputs); + var time_steps = flatted_inptus[0].shape[0]; + var batch = flatted_inptus[0].shape[1]; + var time_step_t = tf.shape(flatted_inptus[0])[0]; + + foreach (var input_ in flatted_inptus) + { + input_.shape.with_rank_at_least(3); + } + + if (mask != null) + { + if (mask.dtype != TF_DataType.TF_BOOL) + { + mask = tf.cast(mask, TF_DataType.TF_BOOL); + } + + if (mask.rank == 2) + { + mask = tf.expand_dims(mask, -1); + } + + if (!time_major) + { + mask = swap_batch_timestep(mask); + } + + } + + if (constants == null) + { + constants = new List(); + } + + // tf.where needs its condition tensor to be the same shape as its two + // result tensors, but in our case the condition (mask) tensor is + // (nsamples, 1), and inputs are (nsamples, ndimensions) or even more. + // So we need to broadcast the mask to match the shape of inputs. + // That's what the tile call does, it just repeats the mask along its + // second dimension n times. + + Tensors _expand_mask(Tensors mask_t, Tensors input_t, int fixed_dim = 1) + { + if (nest.is_nested(mask_t)) + { + throw new ValueError($"mask_t is expected to be tensor, but got {mask_t}"); + } + + if (nest.is_nested(input_t)) + { + throw new ValueError($"input_t is expected to be tensor, but got {input_t}"); + } + + var rank_diff = input_t.rank - mask_t.rank; + for (int i = 0; i < rank_diff; i++) + { + mask_t = tf.expand_dims(mask_t, -1); + } + var multiples = Enumerable.Repeat(1, fixed_dim).ToArray().concat(input_t.shape.as_int_list().ToList().GetRange(fixed_dim, input_t.rank)); + return tf.tile(mask_t, multiples); + } + + Tensors outputs = new Tensors(); + Tensors output_time_zero = new Tensors(); + Tensors last_output = new Tensors(); + Tensors new_states = new Tensors(); + if (unroll) + { + if (time_steps == 0) + { + throw new ValueError("Unrolling requires a fixed number of timesteps."); + } + + // Process the input tensors. The input tensor need to be split on the + // time_step dim, and reverse if go_backwards is True. In the case of + // nested input, the input is flattened and then transformed + // individually. The result of this will be a tuple of lists, each of + // the item in tuple is list of the tensor with shape (batch, feature) + + + // TODO(Wanglongzhi2001)，step_func接受的第二个参数为List，但是最后却用的tuple + //var states = Tuple.Create(initial_states); + var states = initial_states; + + var successive_states = new Tensors(); + var successive_outputs = new Tensors(); + + // Process the input tensors. The input tensor need to be split on the + // time_step dim, and reverse if go_backwards is True. In the case of + // nested input, the input is flattened and then transformed + // individually. The result of this will be a tuple of lists, each of + // the item in tuple is list of the tensor with shape (batch, feature) + + + + + Tensors _process_single_input_t(Tensors input_t) + { + input_t = tf.unstack(input_t); // unstack for time_step dim + if (go_backwards) + { + input_t.Reverse(); + } + return input_t; + } + + // TODO(Wanglongzhi2001) + Tensors processed_input; + if (nest.is_nested(inputs)) + { + processed_input = nest.map_structure(_process_single_input_t, inputs); + } + else + { + processed_input = _process_single_input_t(inputs); + } + + object _get_input_tensor(int time) + { + List inp = new List(); + foreach (var t_ in processed_input) + { + inp.Add(t_[time]); + } + return nest.pack_sequence_as(inputs, inp); + } + + //if (mask != null) + //{ + // var mask_list = tf.unstack(mask); + // if (go_backwards) + // { + // mask_list.Reverse(); + // } + + // for (int i = 0; i < time_steps; i++) + // { + // // TODO(Wanglongzhi2001),deal with _get_input_tensor + // var inp = _get_input_tensor(i); + // var mask_t = mask_list[i]; + // // TODO + // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); + + // var tiled_mask_t = _expand_mask(mask_t, output); + + // Tensors prev_output; + // if (successive_outputs == null) + // { + // prev_output = tf.zeros_like(output); + // } + // else + // { + // prev_output = successive_outputs[successive_outputs.Length - 1]; + // } + + // output = tf.where(tiled_mask_t, output, prev_output); + + // //var flat_states = nest.flatten(states); + // //var flat_new_states = nest.flatten(newStates); + // var flat_states = states.ToList(); + // var flat_new_states = newStates.ToList(); + + // var tiledMaskT = flat_states + // .Select(s => _expand_mask(mask_t, s)) + // .ToArray(); + // var tuple = Tuple.Create(tiledMaskT); + + // List flat_final_states = new List(); + // foreach (var (m, s, ps) in Enumerable.Zip(tiled_mask_t, flat_new_states, flat_states)) + // { + // flat_final_states.Add(tf.where(m, s, ps)); + // } + + // states = (Tensors)nest.pack_sequence_as(states, flat_final_states); + // if (return_all_outputs) + // { + // successive_outputs.Add(output); + // successive_states.Add(states); + // } + // else + // { + // successive_outputs = new Tensors { output }; + // successive_states = new Tensors { states }; + // } + + // } + // last_output = successive_outputs[successive_outputs.Length - 1]; + // new_states = successive_states[successive_states.Length - 1]; + // outputs = tf.stack(successive_outputs); + + // if (zero_output_for_mask) + // { + // last_output = tf.where(_expand_mask(mask_list[mask_list.Length - 1], last_output), last_output, tf.zeros_like(last_output)); + // outputs = tf.where(_expand_mask(mask, outputs, fixed_dim: 2), outputs, tf.zeros_like(outputs)); + // } + // else // mask is null + // { + // for (int i = 0; i < time_steps; i++) + // { + // var inp = _get_input_tensor(i); + // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); + // states = newStates; + + // if (return_all_outputs) + // { + // successive_outputs.Add(output); + // successive_states.Add(newStates); + // } + // else + // { + // successive_outputs = new Tensors { output }; + // successive_states = new Tensors { newStates }; + // } + // } + // last_output = successive_outputs[successive_outputs.Length - 1]; + // new_states = successive_states[successive_states.Length - 1]; + // outputs = tf.stack(successive_outputs); + // } + //} + } + //else // unroll == false + //{ + // var states = initial_states; + // // Create input tensor array, if the inputs is nested tensors, then it + // // will be flattened first, and tensor array will be created one per + // // flattened tensor. + // var input_ta = new List(); + // for (int i = 0; i < flatted_inptus.Count; i++) + // { + // input_ta.Add(tf.TensorArray(dtype: flatted_inptus[i].dtype, size: time_step_t)); + // } + + // // Get the time(0) input and compute the output for that, the output will + // // be used to determine the dtype of output tensor array. Don't read from + // // input_ta due to TensorArray clear_after_read default to True. + // var inps = new Tensors(); + // foreach (var inp in flatted_inptus) + // { + // inps.Add(inp[0]); + // } + // var input_time_zero = nest.pack_sequence_as(inputs, inps); + + // // output_time_zero is used to determine the cell output shape and its + // // dtype. the value is discarded. + // (output_time_zero, _) = step_function((Tensor)input_time_zero, new Tensors { initial_states, constants }); + + // var output_ta_size = return_all_outputs ? time_step_t : tf.constant(1); + // var output_ta = new List(); + // for (int i = 0; i < output_time_zero.ToList().Count; i++) + // { + // var Out = output_time_zero.ToList()[i]; + // output_ta.Add(tf.TensorArray(dtype: Out.dtype, size: output_ta_size, element_shape: Out.shape)); + // } + + // var time = tf.constant(0, dtype: TF_DataType.TF_INT32, name: "time"); + + + + // Func? masking_fn; + // Func? compute_masked_output = null; + // if (mask != null) + // { + // if (go_backwards) + // { + // mask = tf.reverse(mask, axis: new[] { 0 }); + // } + // var mask_ta = tf.TensorArray(dtype: TF_DataType.TF_BOOL, size: time_step_t); + // mask_ta = mask_ta.unstack(mask); + + // masking_fn = (time) => + // { + // return mask_ta.read(time); + // }; + + // compute_masked_output = (mask_t, flat_out, flat_mask) => + // { + // var tiled_mask_t = new Tensors(); + // foreach (var o in flat_out) + // { + // tiled_mask_t.Add(_expand_mask(mask_t, o, fixed_dim: mask_t.rank)); + // } + + // Tensors res = new Tensors(); + // foreach (var (m, o, fm) in Enumerable.Zip(tiled_mask_t, flat_out, flat_mask)) + // { + // res.Add(tf.where(m, o, fm)); + // } + // return res; + // }; + // } + // // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor)? + // else if (input_length is Tensor) + // { + // if (go_backwards) + // { + // var max_len = tf.reduce_max(input_length, axis: 0); + // var rev_input_length = tf.subtract(max_len - 1, input_length); + + // masking_fn = (time) => + // { + // return tf.less(rev_input_length, time); + // }; + // } + // else + // { + // masking_fn = (time) => + // { + // return tf.greater(input_length, time); + // }; + // } + + // compute_masked_output = (mask_t, flat_out, flat_mask) => + // { + // var res = new List(); + // foreach (var (o, zo) in zip(flat_out, flat_mask)) + // { + // res.Add(tf.where(mask_t, o, zo)); + // } + // return res; + // }; + // } + // else + // { + // masking_fn = null; + // } + + + // if (masking_fn != null) + // { + // // Mask for the T output will be base on the output of T - 1. In the + // // case T = 0, a zero filled tensor will be used. + // var flat_zero_output = new Tensors(); + // foreach (var o in nest.flatten(output_time_zero)) + // { + // flat_zero_output.Add(tf.zeros_like(o)); + // } + + + // (Tensor, List, Tensors, Tensors) _step(Tensor time, List output_ta_t, Tensors prev_output, Tensors states) + // { + // /* + // RNN step function. + // Args: + // time: Current timestep value. + // output_ta_t: TensorArray. + // prev_output: tuple of outputs from time - 1. + // *states: List of states. + // Returns: + // Tuple(todo): `(time + 1, output_ta_t, output) + tuple(new_states)` + // */ + + // var current_input = input_ta.Select(x => x.read(time)).ToList(); + // // maybe set shape + // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + // current_input = (List)nest.pack_sequence_as(inputs, current_input); + // var mask_t = masking_fn(time); + // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); + // // mask output + // //var flat_output = nest.flatten(output); + // var flat_output = output.ToList(); + + // var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.ToList(); + + // // TODO(Wanglongzhi2001),deal with compute_masked_output's third parameter's type + // var flat_new_output = compute_masked_output(mask_t, flat_output, flat_mask_output); + + // // mask states + // var flat_state = states.ToList(); + // var flat_new_state = new_states.ToList(); + + // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + // { + // if (new_state is Tensor) + // { + // new_state.set_shape(state.shape); + // } + // } + + // var flat_final_state = compute_masked_output(mask_t, flat_new_state, flat_state); + // new_states = (Tensors)nest.pack_sequence_as(new_states, flat_final_state); + + // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + // var Output_ta_t = new List(); + // // TODO(Wanglongzhi2001),deal with zip output_ta_t + // foreach (var (ta, Out) in zip(output_ta_t, flat_new_output)) + // { + // Output_ta_t.Add(ta.write(ta_index_to_write, Out)); + // } + + + + // //new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); + + + // return (time + 1, Output_ta_t, flat_new_output, new_states); + + // } + // Func cond = (time) => (time < time_step_t); + + // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, flat_zero_output, states)); + // new_states = final_outputs.Item4; + // output_ta = final_outputs.Item2; + + // } + // else + // { + // (Tensor, List, Tensors) _step(Tensor time, List output_ta_t, Tensors states) + // { + // var current_input = input_ta.Select(x => x.read(time)).ToList(); + // // maybe set shape + // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + // current_input = (List)nest.pack_sequence_as(inputs, current_input); + // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); + // var flat_state = states.ToList(); + // var flat_new_state = new_states.ToList(); + // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + // { + // if (new_state is Tensor) + // { + // new_state.set_shape(state.shape); + // } + // } + // var flat_output = output.ToList(); + // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + // var Output_ta_t = new List(); + // foreach (var (ta, out_) in zip(output_ta_t, flat_output)) + // { + // Output_ta_t.Add(ta.write(ta_index_to_write, out_)); + // } + + // new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); + // return (time + 1, Output_ta_t, new_states); + // } + // Func cond = (time) => (time < time_step_t); + // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, states)); + // new_states = final_outputs.Item3; + // output_ta = final_outputs.Item2; + + // } + // //Tensors outputs = new Tensors(); + // foreach (var o in output_ta) + // { + // outputs.Add(o.stack()); + // } + // foreach (var o in outputs) + // { + // last_output.Add(o[-1]); + // } + // outputs = (Tensors)nest.pack_sequence_as(output_time_zero, outputs); + // last_output = (Tensors)nest.pack_sequence_as(output_time_zero, last_output); + + //} + + Func set_shape; + set_shape = (output_) => + { + if (output_ is Tensor) + { + var shape = output_.shape.as_int_list(); + if (return_all_outputs) + { + shape[0] = (int)time_steps; + } + else + { + shape[0] = 1; + } + shape[1] = (int)batch; + output_.set_shape(new Tensor(shape)); + } + return output_; + }; + + var Outputs = (Tensors)nest.map_structure(set_shape, outputs); + if (!time_major) + { + Outputs = nest.map_structure(swap_batch_timestep, outputs); + } + return (last_output, Outputs, new_states); + + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index e768bd0b..7347585f 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving.SavedModel; using Tensorflow.Keras.Utils; @@ -81,7 +82,7 @@ namespace Tensorflow.Keras.Engine } else { - _buildInputShape = new Saving.TensorShapeConfig(); + _buildInputShape = new TensorShapeConfig(); } if (outputs.Any(x => x.KerasHistory == null)) @@ -325,7 +326,7 @@ namespace Tensorflow.Keras.Engine nodes_in_decreasing_depth.append(node); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var tensor_dict = new Dictionary>(); // map input values diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index c0430458..a0358f07 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -1,4 +1,5 @@ using System.Threading; +using Tensorflow.Common.Types; using static Tensorflow.Binding; namespace Tensorflow.Keras.Engine @@ -8,11 +9,11 @@ namespace Tensorflow.Keras.Engine /// /// Wraps `call`, applying pre- and post-processing steps. /// - /// + /// /// /// /// - public Tensors Apply(Tensors inputs, Tensor state = null, bool training = false) + public virtual Tensors Apply(Tensors inputs, Tensors states = null, bool training = false, IOptionalArgs? optional_args = null) { if (callContext.Value == null) callContext.Value = new CallContext(); @@ -30,7 +31,7 @@ namespace Tensorflow.Keras.Engine if (!built) MaybeBuild(inputs); - var outputs = Call(inputs, state: state, training: training); + var outputs = Call(inputs, state: states, training: training); // memory leak // _set_connectivity_metadata_(inputs, outputs); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index 5942efd9..2f758a85 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -32,7 +32,7 @@ using Tensorflow.Util; using static Tensorflow.Binding; using Tensorflow.Framework; using Tensorflow.Sessions; - +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Engine { @@ -332,7 +332,7 @@ namespace Tensorflow.Keras.Engine /// /// /// - protected virtual Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected virtual Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if(ReplacedCall is not null) { diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index 83702b23..7b35d547 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -1,8 +1,8 @@ using System.Diagnostics; +using Tensorflow.Common.Types; using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Losses; -using Tensorflow.Keras.Saving; using Tensorflow.Keras.Saving.SavedModel; using Tensorflow.Keras.Utils; using Tensorflow.Train; diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 27874751..6a468ad2 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -21,6 +21,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Utils; using static Tensorflow.KerasApi; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Engine { @@ -143,7 +144,7 @@ namespace Tensorflow.Keras.Engine } } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (!_has_explicit_input_shape) { diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs index 739c0d56..23f36c86 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -29,7 +30,7 @@ namespace Tensorflow.Keras.Layers { base.build(input_shape); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; output = tf.where(output > 0f, output, diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs index 17636302..81fefb31 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs @@ -4,7 +4,7 @@ using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; -using static Tensorflow.Binding; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { public class Exponential : Layer @@ -17,7 +17,7 @@ namespace Tensorflow.Keras.Layers { { base.build(input_shape); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; return tf.exp(output); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs b/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs index b498d1b9..e0f91380 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/HardSigmoid.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers { @@ -10,7 +11,7 @@ namespace Tensorflow.Keras.Layers { public HardSigmoid ( LayerArgs args ) : base(args) { // hard sigmoid has no arguments } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null ) { Tensor x = inputs; return tf.clip_by_value( tf.add(tf.multiply(x, 0.2f), 0.5f), 0f, 1f); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs b/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs index 1fbbf4ea..cfbd0186 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/LeakyReLu.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers @@ -19,7 +20,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return tf.nn.leaky_relu(inputs, alpha: alpha); } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs index 53101fbb..2e943d5f 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -22,7 +23,7 @@ namespace Tensorflow.Keras.Layers { } base.build(input_shape); } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; return tf.where(output > 0f, tf.multiply(scale, output), diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs index 3ffae27f..d018128d 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softmax.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using static Tensorflow.Binding; @@ -11,8 +12,8 @@ namespace Tensorflow.Keras.Layers { public Softmax ( SoftmaxArgs args ) : base(args) { axis = args.axis; } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor x = inputs.Length == 2 ? inputs + ((1.0 - tf.cast(inputs[1], inputs.dtype)) * 1e-9) + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { + Tensor x = inputs.Length == 2 ? inputs[0] + ((1.0 - tf.cast(inputs[1], inputs.dtype)) * 1e-9) : inputs; Tensor e = tf.exp(tf.sub(x, tf.reduce_max(x, axis: this.axis, keepdims: true))); Tensor s = tf.reduce_sum(e, axis: this.axis, keepdims: true); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs index e82b0198..1e6c59b4 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softplus.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using static Tensorflow.Binding; @@ -10,7 +11,7 @@ namespace Tensorflow.Keras.Layers { public Softplus ( LayerArgs args ) : base(args) { // Softplus has no arguments } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor x = inputs; return tf.log( tf.add(tf.exp(x), 1f)); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs b/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs index 59329fd4..5ad33e99 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Softsign.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using static Tensorflow.Binding; @@ -10,7 +11,7 @@ namespace Tensorflow.Keras.Layers { public Softsign ( LayerArgs args ) : base(args) { // Softsign has no arguments } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor x = inputs; // x / (abs(x) + 1) return tf.div(x, tf.add(1f, tf.abs(x))); diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs b/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs index 1dcb92b3..ed0d105a 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Swish.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using static Tensorflow.Binding; @@ -10,7 +11,7 @@ namespace Tensorflow.Keras.Layers { public Swish ( LayerArgs args ) : base(args) { // Swish has no arguments } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { + protected override Tensors Call ( Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor x = inputs; // x / (1 + exp(-x)) diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs b/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs index 99b80394..7e90cf9d 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Tanh.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers @@ -13,7 +14,7 @@ namespace Tensorflow.Keras.Layers { // Tanh has no arguments } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor x = inputs; diff --git a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs index 1348e19c..19b29272 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Linq; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; /// /// Base class for attention layers that can be used in sequence DNN/CNN models. @@ -114,7 +115,7 @@ namespace Tensorflow.Keras.Layers return (tf.linalg.einsum("bij,bjk->bik", (weights, value)), weights); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensors _inp; Tensors _mask = null; diff --git a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs index 701724d5..75dd4a41 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs @@ -6,6 +6,7 @@ using static Tensorflow.Binding; using static Tensorflow.KerasApi; using System; using System.Linq; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -252,7 +253,7 @@ namespace Tensorflow.Keras.Layers return (attention_output, attention_scores); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensors _inp; Tensor _mask = null; @@ -349,7 +350,7 @@ namespace Tensorflow.Keras.Layers //} if (return_attention_scores) - return (attention_output, attention_scores); + return (attention_output, attention_scores.Single); return attention_output; } } diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs index bbd49acd..94ad7914 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs @@ -20,6 +20,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Utils; using static Tensorflow.KerasApi; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -83,7 +84,7 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var inputs_shape = array_ops.shape(inputs); var batch_size = inputs_shape[0]; diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs index c575362c..d8e00d52 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -103,7 +104,7 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = false) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = false, IOptionalArgs? optional_args = null) { var outputs = _convolution_op.Apply(inputs, kernel.AsTensor()); if (use_bias) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index aa6617dd..db5d626e 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -69,7 +70,7 @@ namespace Tensorflow.Keras.Layers built = true; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor outputs = null; var rank = inputs.rank; diff --git a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs index fb604f77..0cbd5084 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs @@ -7,6 +7,7 @@ using System.Text.RegularExpressions; using Tensorflow.Keras.Engine; using Tensorflow.Keras.ArgsDefinition.Core; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -189,7 +190,7 @@ namespace Tensorflow.Keras.Layers // return new dict(base_config.items().ToList() + config.items().ToList()); //} - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var ret = tf.linalg.einsum(this.equation, (inputs, this.kernel.AsTensor())); if (this.bias != null) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs index 9487a7d0..87b42bb7 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs @@ -15,6 +15,7 @@ ******************************************************************************/ using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -66,7 +67,7 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var dtype = inputs.dtype; if (dtype != tf.int32 && dtype != tf.int64) diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs index 7df654ee..bcbb20d8 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs @@ -5,6 +5,7 @@ using static Tensorflow.Binding; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -21,7 +22,7 @@ namespace Tensorflow.Keras.Layers _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return _merge_function(inputs); } diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs index d02d2509..65558157 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -146,7 +147,7 @@ namespace Tensorflow.Keras.Layers return false; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor outputs = null; var training_tensor = training == null diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs index e90c0402..1898f24c 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -101,7 +102,7 @@ namespace Tensorflow.Keras.Layers return input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor outputs = null; var inputs_dtype = inputs.dtype.as_base_dtype(); diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs index a65154bf..987b56bc 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/Normalization.cs @@ -14,6 +14,7 @@ limitations under the License. ******************************************************************************/ +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; @@ -157,7 +158,7 @@ namespace Tensorflow.Keras.Layers base.adapt(data, batch_size: batch_size, steps: steps); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (_args.Invert) { diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs index d62fb63a..ffaabec9 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling1D.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -12,7 +13,7 @@ namespace Tensorflow.Keras.Layers { } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (data_format == "channels_last") return math_ops.reduce_mean(inputs, 1, false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs index 000e4b8b..e0666517 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalAveragePooling2D.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -12,7 +13,7 @@ namespace Tensorflow.Keras.Layers { } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (data_format == "channels_last") return math_ops.reduce_mean(inputs, (1, 2), false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs index 2de4671c..15695e8a 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling1D.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -12,7 +13,7 @@ namespace Tensorflow.Keras.Layers { } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (data_format == "channels_last") return math_ops.reduce_max(inputs, 1, false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs index b7e2c945..76db858d 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/GlobalMaxPooling2D.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -12,7 +13,7 @@ namespace Tensorflow.Keras.Layers { } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (data_format == "channels_last") return math_ops.reduce_max(inputs, (1, 2), false); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs index a2f4c51b..81a34019 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs @@ -18,6 +18,7 @@ using System.Linq; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; +using Tensorflow.Common.Types; using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers @@ -36,7 +37,7 @@ namespace Tensorflow.Keras.Layers input_spec = new InputSpec(ndim: 3); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { int pad_axis = args.DataFormat == "channels_first" ? 2 : 3; inputs = tf.expand_dims(inputs, pad_axis); diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs index 27032255..f83f1e15 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs @@ -17,6 +17,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -36,7 +37,7 @@ namespace Tensorflow.Keras.Layers input_spec = new InputSpec(ndim: 4); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { int[] pool_shape; int[] strides; diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs index 5620a916..20d2a53d 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs @@ -1,6 +1,6 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; - +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { /// @@ -15,7 +15,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var depth = args.NumTokens; var max_value = tf.reduce_max(inputs); diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs index 5fc581af..7fa367ee 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs @@ -1,5 +1,6 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -17,7 +18,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { scale = constant_op.constant(args.Scale, args.DType); offset = constant_op.constant(args.Offset, args.DType); diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs index 603e2b07..081966ad 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/Resizing.cs @@ -4,6 +4,7 @@ using System; using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -19,7 +20,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return image_ops_impl.resize_images_v2(inputs, new[] { args.Height, args.Width }, method: args.Interpolation); } diff --git a/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs b/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs index aa3a92a4..ada1851c 100644 --- a/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs +++ b/src/TensorFlowNET.Keras/Layers/Regularization/Dropout.cs @@ -1,4 +1,5 @@ -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; using static Tensorflow.Binding; @@ -15,7 +16,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (training == null) training = false; diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs index 9ead15cb..31285438 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs @@ -1,6 +1,8 @@ using Tensorflow.Keras.ArgsDefinition.Reshaping; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Reshaping { @@ -27,7 +29,7 @@ namespace Tensorflow.Keras.Layers.Reshaping _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; if (output.rank != 3) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs index 087d59a1..4a5c6eab 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs @@ -1,6 +1,7 @@ using Tensorflow.Keras.ArgsDefinition.Reshaping; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Reshaping { @@ -21,7 +22,7 @@ namespace Tensorflow.Keras.Layers.Reshaping built = true; _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; if (output.rank != 4) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs index 04a1af60..83f86c6f 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs @@ -1,6 +1,7 @@ using Tensorflow.Keras.ArgsDefinition.Reshaping; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Reshaping { @@ -21,7 +22,7 @@ namespace Tensorflow.Keras.Layers.Reshaping _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor output = inputs; if (output.rank != 5) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs index 539b5f62..a6192849 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Flatten.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Framework; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; @@ -23,7 +24,7 @@ namespace Tensorflow.Keras.Layers _channels_first = args.DataFormat == "channels_first"; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (_channels_first) { diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs index e391775c..7fdb816b 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs @@ -6,6 +6,7 @@ using Tensorflow.Keras.Utils; using static Tensorflow.Binding; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { public class Permute : Layer @@ -28,7 +29,7 @@ namespace Tensorflow.Keras.Layers { built = true; _buildInputShape = input_shape; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { Tensor outputs = inputs; return tf.transpose(outputs, new Axis(permute)); diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs index 92a772f3..4b3d30e2 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Reshape.cs @@ -4,6 +4,7 @@ using static Tensorflow.Binding; using System.Collections.Generic; using System; using System.Linq; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -19,7 +20,7 @@ namespace Tensorflow.Keras.Layers this.args = args; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { var shapes = new List(); shapes.Add(array_ops.shape(inputs)[0]); diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs index 8314151f..223f33d4 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs @@ -6,6 +6,7 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; using static Tensorflow.Binding; using static Tensorflow.KerasApi; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -24,7 +25,7 @@ namespace Tensorflow.Keras.Layers inputSpec = new InputSpec(ndim: 4); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return keras.backend.resize_images(inputs, size[0], size[1], diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs index 7c87100a..3b37dac4 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/ZeroPadding2D.cs @@ -2,6 +2,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; +using Tensorflow.Common.Types; using static Tensorflow.KerasApi; namespace Tensorflow.Keras.Layers @@ -26,7 +27,7 @@ namespace Tensorflow.Keras.Layers this.input_spec = new InputSpec(ndim: 4); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { return keras.backend.spatial_2d_padding(inputs, padding: padding, diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs new file mode 100644 index 00000000..21396853 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public abstract class DropoutRNNCellMixin: RnnCellBase + { + public float dropout; + public float recurrent_dropout; + // TODO(Rinne): deal with cache. + public DropoutRNNCellMixin(LayerArgs args): base(args) + { + + } + + public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + { + if (dropout == 0f) + return null; + return _generate_dropout_mask( + tf.ones_like(input), + dropout, + training, + count); + } + + // Get the recurrent dropout mask for RNN cell. + public Tensors? get_recurrent_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + { + if (dropout == 0f) + return null; + return _generate_dropout_mask( + tf.ones_like(input), + recurrent_dropout, + training, + count); + } + + public Tensors _create_dropout_mask(Tensors input, bool training, int count = 1) + { + return _generate_dropout_mask( + tf.ones_like(input), + dropout, + training, + count); + } + + public Tensors _create_recurrent_dropout_mask(Tensors input, bool training, int count = 1) + { + return _generate_dropout_mask( + tf.ones_like(input), + recurrent_dropout, + training, + count); + } + + public Tensors _generate_dropout_mask(Tensor ones, float rate, bool training, int count = 1) + { + Tensors dropped_inputs() + { + DropoutArgs args = new DropoutArgs(); + args.Rate = rate; + var DropoutLayer = new Dropout(args); + var mask = DropoutLayer.Apply(ones, training: training); + return mask; + } + + if (count > 1) + { + Tensors results = new Tensors(); + for (int i = 0; i < count; i++) + { + results.Add(dropped_inputs()); + } + return results; + } + + return dropped_inputs(); + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 59555e62..1449c908 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -1,6 +1,7 @@ using System.Linq; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Rnn { @@ -26,9 +27,9 @@ namespace Tensorflow.Keras.Layers.Rnn .ToArray(); } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { - return base.Call(inputs, state: state, training: training); + return base.Call(inputs, initial_state: state, training: training); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 310e8057..b014737f 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -1,53 +1,466 @@ -using System; +using OneOf; +using System; using System.Collections.Generic; +using System.Reflection; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Util; +using Tensorflow.Common.Extensions; +using System.Linq.Expressions; +using Tensorflow.Keras.Utils; +using Tensorflow.Common.Types; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; namespace Tensorflow.Keras.Layers.Rnn { - public class RNN : Layer + /// + /// Base class for recurrent layers. + /// See [the Keras RNN API guide](https://www.tensorflow.org/guide/keras/rnn) + /// for details about the usage of RNN API. + /// + public class RNN : RnnBase { - private RNNArgs args; - private object input_spec = null; // or NoneValue?? - private object state_spec = null; - private object _states = null; - private object constants_spec = null; - private int _num_constants = 0; - protected IVariableV1 kernel; - protected IVariableV1 bias; - protected ILayer cell; + private RNNArgs _args; + private object _input_spec = null; // or NoneValue?? + private object _state_spec = null; + private Tensors _states = null; + private object _constants_spec = null; + private int _num_constants; + protected IVariableV1 _kernel; + protected IVariableV1 _bias; + protected IRnnCell _cell; + public RNN(RNNArgs args) : base(PreConstruct(args)) { - this.args = args; + _args = args; SupportsMasking = true; - // The input shape is unknown yet, it could have nested tensor inputs, and - // the input spec will be the list of specs for nested inputs, the structure - // of the input_spec will be the same as the input. + // if is StackedRnncell + _cell = args.Cell; - //if(stateful) - //{ - // if (ds_context.has_strategy()) // ds_context???? - // { - // throw new Exception("RNNs with stateful=True not yet supported with tf.distribute.Strategy"); - // } - //} + // get input_shape + _args = PreConstruct(args); + + _num_constants = 0; + } + + // States is a tuple consist of cell states_size, like (cell1.state_size, cell2.state_size,...) + // state_size can be a single integer, can also be a list/tuple of integers, can also be TensorShape or a list/tuple of TensorShape + public Tensors States + { + get + { + if (_states == null) + { + // CHECK(Rinne): check if this is correct. + var state = nest.map_structure(x => null, _cell.StateSize); + return new Tensors { state }; + } + return _states; + } + set { _states = value; } + } + + private OneOf> compute_output_shape(Shape input_shape) + { + var batch = input_shape[0]; + var time_step = input_shape[1]; + if (_args.TimeMajor) + { + (batch, time_step) = (time_step, batch); + } + + // state_size is a array of ints or a positive integer + var state_size = _cell.StateSize.ToSingleShape(); + + // TODO(wanglongzhi2001),flat_output_size应该是什么类型的，Shape还是Tensor + Func _get_output_shape; + _get_output_shape = (flat_output_size) => + { + var output_dim = flat_output_size.as_int_list(); + Shape output_shape; + if (_args.ReturnSequences) + { + if (_args.TimeMajor) + { + output_shape = new Shape(new int[] { (int)time_step, (int)batch }.concat(output_dim)); + } + else + { + output_shape = new Shape(new int[] { (int)batch, (int)time_step }.concat(output_dim)); + + } + } + else + { + output_shape = new Shape(new int[] { (int)batch }.concat(output_dim)); + } + return output_shape; + }; + + Type type = _cell.GetType(); + PropertyInfo output_size_info = type.GetProperty("output_size"); + Shape output_shape; + if (output_size_info != null) + { + output_shape = nest.map_structure(_get_output_shape, _cell.OutputSize.ToSingleShape()); + // TODO(wanglongzhi2001),output_shape应该简单的就是一个元组还是一个Shape类型 + output_shape = (output_shape.Length == 1 ? (int)output_shape[0] : output_shape); + } + else + { + output_shape = _get_output_shape(state_size); + } + + if (_args.ReturnState) + { + Func _get_state_shape; + _get_state_shape = (flat_state) => + { + var state_shape = new int[] { (int)batch }.concat(flat_state.as_int_list()); + return new Shape(state_shape); + }; + var state_shape = _get_state_shape(state_size); + + return new List { output_shape, state_shape }; + } + else + { + return output_shape; + } + + } + + private Tensors compute_mask(Tensors inputs, Tensors mask) + { + // Time step masks must be the same for each input. + // This is because the mask for an RNN is of size [batch, time_steps, 1], + // and specifies which time steps should be skipped, and a time step + // must be skipped for all inputs. + + mask = nest.flatten(mask)[0]; + var output_mask = _args.ReturnSequences ? mask : null; + if (_args.ReturnState) + { + var state_mask = new List(); + for (int i = 0; i < len(States); i++) + { + state_mask.Add(null); + } + return new List { output_mask }.concat(state_mask); + } + else + { + return output_mask; + } } public override void build(KerasShapesWrapper input_shape) { - if (!cell.Built) + object get_input_spec(Shape shape) + { + var input_spec_shape = shape.as_int_list(); + + var (batch_index, time_step_index) = _args.TimeMajor ? (1, 0) : (0, 1); + if (!_args.Stateful) + { + input_spec_shape[batch_index] = -1; + } + input_spec_shape[time_step_index] = -1; + return new InputSpec(shape: input_spec_shape); + } + + Shape get_step_input_shape(Shape shape) + { + + // return shape[1:] if self.time_major else (shape[0],) + shape[2:] + if (_args.TimeMajor) + { + return shape.as_int_list().ToList().GetRange(1, shape.Length - 1).ToArray(); + } + else + { + return new int[] { shape.as_int_list()[0] }.concat(shape.as_int_list().ToList().GetRange(2, shape.Length - 2).ToArray()); + } + + + } + + object get_state_spec(Shape shape) + { + var state_spec_shape = shape.as_int_list(); + // append bacth dim + state_spec_shape = new int[] { -1 }.concat(state_spec_shape); + return new InputSpec(shape: state_spec_shape); + + } + + // Check whether the input shape contains any nested shapes. It could be + // (tensor_shape(1, 2), tensor_shape(3, 4)) or (1, 2, 3) which is from + // numpy inputs. + + + if (!_cell.Built) { - cell.build(input_shape); + _cell.build(input_shape); } } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + /// + /// + /// + /// + /// Binary tensor of shape [batch_size, timesteps] indicating whether a given timestep should be masked + /// + /// List of initial state tensors to be passed to the first call of the cell + /// List of constant tensors to be passed to the cell at each timestep + /// + /// + /// + protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bool? training = null, IOptionalArgs? optional_args = null) { - return base.Call(inputs, state, training); + RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; + if(optional_args is not null && rnn_optional_args is null) + { + throw new ArgumentException("The optional args shhould be of type `RnnOptionalArgs`"); + } + Tensors? constants = rnn_optional_args?.Constants; + Tensors? mask = rnn_optional_args?.Mask; + //var (inputs_padded, row_length) = BackendImpl.convert_inputs_if_ragged(inputs); + // 暂时先不接受ragged tensor + int? row_length = null; + bool is_ragged_input = false; + _validate_args_if_ragged(is_ragged_input, mask); + + (inputs, initial_state, constants) = _process_inputs(inputs, initial_state, constants); + + _maybe_reset_cell_dropout_mask(_cell); + if (_cell is StackedRNNCells) + { + var stack_cell = _cell as StackedRNNCells; + foreach (var cell in stack_cell.Cells) + { + _maybe_reset_cell_dropout_mask(cell); + } + } + + if (mask != null) + { + // Time step masks must be the same for each input. + mask = nest.flatten(mask)[0]; + } + + Shape input_shape; + if (nest.is_nested(inputs)) + { + // In the case of nested input, use the first element for shape check + // input_shape = nest.flatten(inputs)[0].shape; + // TODO(Wanglongzhi2001) + input_shape = nest.flatten(inputs)[0].shape; + } + else + { + input_shape = inputs.shape; + } + + var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; + + if (_args.Unroll && timesteps != null) + { + throw new ValueError( + "Cannot unroll a RNN if the " + + "time dimension is undefined. \n" + + "- If using a Sequential model, " + + "specify the time dimension by passing " + + "an `input_shape` or `batch_input_shape` " + + "argument to your first layer. If your " + + "first layer is an Embedding, you can " + + "also use the `input_length` argument.\n" + + "- If using the functional API, specify " + + "the time dimension by passing a `shape` " + + "or `batch_shape` argument to your Input layer." + ); + } + + // cell_call_fn = (self.cell.__call__ if callable(self.cell) else self.cell.call) + Func step; + if (constants is not null) + { + if (!_cell.SupportOptionalArgs) + { + throw new ValueError( + $"RNN cell {_cell} does not support constants." + + $"Received: constants={constants}"); + } + + step = (inputs, states) => + { + constants = new Tensors(states.TakeLast(_num_constants)); + states = new Tensors(states.SkipLast(_num_constants)); + var(output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); + // TODO(Wanglongzhi2001),should cell_call_fn's return value be Tensors, Tensors? + return (output, new_states.Single); + }; + } + else + { + step = (inputs, states) => + { + // states = (states[0] if len(states) == 1 and is_tf_rnn_cell else states) + var (output, new_states) = _cell.Apply(inputs, states); + return (output, new_states.Single); + }; + } + + var (last_output, outputs, states) = BackendImpl.rnn(step, + inputs, + initial_state, + constants: constants, + go_backwards: _args.GoBackwards, + mask: mask, + unroll: _args.Unroll, + input_length: row_length != null ? new Tensor(row_length) : new Tensor(timesteps), + time_major: _args.TimeMajor, + zero_output_for_mask: _args.ZeroOutputForMask, + return_all_outputs: _args.ReturnSequences); + + if (_args.Stateful) + { + throw new NotImplementedException("this argument havn't been developed."); + } + + Tensors output = new Tensors(); + if (_args.ReturnSequences) + { + throw new NotImplementedException("this argument havn't been developed."); + + } + else + { + output = last_output; + } + + if (_args.ReturnState) + { + foreach (var state in states) + { + output.Add(state); + } + return output; + } + else + { + return output; + } + } + + public override Tensors Apply(Tensors inputs, Tensors initial_states = null, bool training = false, IOptionalArgs? optional_args = null) + { + RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; + if (optional_args is not null && rnn_optional_args is null) + { + throw new ArgumentException("The type of optional args should be `RnnOptionalArgs`."); + } + Tensors? constants = rnn_optional_args?.Constants; + (inputs, initial_states, constants) = RnnUtils.standardize_args(inputs, initial_states, constants, _num_constants); + + if(initial_states is null && constants is null) + { + return base.Apply(inputs); + } + + // TODO(Rinne): implement it. + throw new NotImplementedException(); + } + + private (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensors inputs, Tensors initial_state, Tensors constants) + { + if (inputs.Length > 1) + { + if (_num_constants != 0) + { + initial_state = new Tensors(inputs.Skip(1)); + } + else + { + initial_state = new Tensors(inputs.Skip(1).SkipLast(_num_constants)); + constants = new Tensors(inputs.TakeLast(_num_constants)); + } + if (len(initial_state) == 0) + initial_state = null; + inputs = inputs[0]; + } + + if (_args.Stateful) + { + if (initial_state != null) + { + var tmp = new Tensor[] { }; + foreach (var s in nest.flatten(States)) + { + tmp.add(tf.math.count_nonzero((Tensor)s)); + } + var non_zero_count = tf.add_n(tmp); + //initial_state = tf.cond(non_zero_count > 0, () => States, () => initial_state); + if ((int)non_zero_count.numpy() > 0) + { + initial_state = States; + } + } + else + { + initial_state = States; + } + + } + else if (initial_state is null) + { + initial_state = get_initial_state(inputs); + } + + if (initial_state.Length != States.Length) + { + throw new ValueError( + $"Layer {this} expects {States.Length} state(s), " + + $"but it received {initial_state.Length} " + + $"initial state(s). Input received: {inputs}"); + } + + return (inputs, initial_state, constants); + } + + private void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) + { + if (!is_ragged_input) + { + return; + } + + if (_args.Unroll) + { + throw new ValueError("The input received contains RaggedTensors and does " + + "not support unrolling. Disable unrolling by passing " + + "`unroll=False` in the RNN Layer constructor."); + } + if (mask != null) + { + throw new ValueError($"The mask that was passed in was {mask}, which " + + "cannot be applied to RaggedTensor inputs. Please " + + "make sure that there is no mask injected by upstream " + + "layers."); + } + + } + + void _maybe_reset_cell_dropout_mask(ILayer cell) + { + //if (cell is DropoutRNNCellMixin) + //{ + // cell.reset_dropout_mask(); + // cell.reset_recurrent_dropout_mask(); + //} } private static RNNArgs PreConstruct(RNNArgs args) @@ -77,60 +490,72 @@ namespace Tensorflow.Keras.Layers.Rnn return args; } - public RNN New(LayerRnnCell cell, - bool return_sequences = false, - bool return_state = false, - bool go_backwards = false, - bool stateful = false, - bool unroll = false, - bool time_major = false) - => new RNN(new RNNArgs - { - Cell = cell, - ReturnSequences = return_sequences, - ReturnState = return_state, - GoBackwards = go_backwards, - Stateful = stateful, - Unroll = unroll, - TimeMajor = time_major - }); - - public RNN New(IList cell, - bool return_sequences = false, - bool return_state = false, - bool go_backwards = false, - bool stateful = false, - bool unroll = false, - bool time_major = false) - => new RNN(new RNNArgs - { - Cell = new StackedRNNCells(new StackedRNNCellsArgs { Cells = cell }), - ReturnSequences = return_sequences, - ReturnState = return_state, - GoBackwards = go_backwards, - Stateful = stateful, - Unroll = unroll, - TimeMajor = time_major - }); - - - protected Tensor get_initial_state(Tensor inputs) + public Tensors __call__(Tensors inputs, Tensor state = null, Tensor training = null) { - return _generate_zero_filled_state_for_cell(null, null); + throw new NotImplementedException(); } - Tensor _generate_zero_filled_state_for_cell(LSTMCell cell, Tensor batch_size) + // 好像不能cell不能传接口类型 + //public RNN New(IRnnArgCell cell, + // bool return_sequences = false, + // bool return_state = false, + // bool go_backwards = false, + // bool stateful = false, + // bool unroll = false, + // bool time_major = false) + // => new RNN(new RNNArgs + // { + // Cell = cell, + // ReturnSequences = return_sequences, + // ReturnState = return_state, + // GoBackwards = go_backwards, + // Stateful = stateful, + // Unroll = unroll, + // TimeMajor = time_major + // }); + + //public RNN New(List cell, + // bool return_sequences = false, + // bool return_state = false, + // bool go_backwards = false, + // bool stateful = false, + // bool unroll = false, + // bool time_major = false) + // => new RNN(new RNNArgs + // { + // Cell = cell, + // ReturnSequences = return_sequences, + // ReturnState = return_state, + // GoBackwards = go_backwards, + // Stateful = stateful, + // Unroll = unroll, + // TimeMajor = time_major + // }); + + + protected Tensors get_initial_state(Tensors inputs) { - throw new NotImplementedException(""); + var input = inputs[0]; + var input_shape = input.shape; + var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; + var dtype = input.dtype; + Tensors init_state; + if (_cell is RnnCellBase rnn_base_cell) + { + init_state = rnn_base_cell.GetInitialState(null, batch_size, dtype); + } + else + { + init_state = RnnUtils.generate_zero_filled_state(batch_size, _cell.StateSize, dtype); + } + + return init_state; } // Check whether the state_size contains multiple states. - public static bool _is_multiple_state(object state_size) + public static bool is_multiple_state(GeneralizedTensorShape state_size) { - var myIndexerProperty = state_size.GetType().GetProperty("Item"); - return myIndexerProperty != null - && myIndexerProperty.GetIndexParameters().Length == 1 - && !(state_size.GetType() == typeof(Shape)); + return state_size.Shapes.Length > 1; } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs new file mode 100644 index 00000000..018b1778 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public abstract class RnnBase: Layer + { + public RnnBase(LayerArgs args): base(args) { } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs new file mode 100644 index 00000000..fcb5d1eb --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Utils; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public abstract class RnnCellBase: Layer, IRnnCell + { + public RnnCellBase(LayerArgs args) : base(args) { } + public abstract GeneralizedTensorShape StateSize { get; } + public abstract GeneralizedTensorShape OutputSize { get; } + public abstract bool SupportOptionalArgs { get; } + public abstract (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null); + public virtual Tensors GetInitialState(Tensors inputs, long batch_size, TF_DataType dtype) + { + return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index 2d7aab70..22d0e277 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -10,18 +10,36 @@ namespace Tensorflow.Keras.Layers.Rnn public class SimpleRNN : RNN { SimpleRNNArgs args; - public SimpleRNN(SimpleRNNArgs args) : base(args) + public SimpleRNN(SimpleRNNArgs args) : base(CreateCellForArgs(args)) { this.args = args; } + private static SimpleRNNArgs CreateCellForArgs(SimpleRNNArgs args) + { + args.Cell = new SimpleRNNCell(new SimpleRNNCellArgs() + { + Units = args.Units, + Activation = args.Activation, + UseBias = args.UseBias, + KernelInitializer = args.KernelInitializer, + RecurrentInitializer = args.RecurrentInitializer, + BiasInitializer = args.BiasInitializer, + Dropout = args.Dropout, + RecurrentDropout = args.RecurrentDropout, + DType = args.DType, + Trainable = args.Trainable, + }); + return args; + } + public override void build(KerasShapesWrapper input_shape) { var single_shape = input_shape.ToSingleShape(); var input_dim = single_shape[-1]; _buildInputShape = input_shape; - kernel = add_weight("kernel", (single_shape[-1], args.Units), + _kernel = add_weight("kernel", (single_shape[-1], args.Units), initializer: args.KernelInitializer //regularizer = self.kernel_regularizer, //constraint = self.kernel_constraint, diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 46061b21..abb57d8a 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -4,47 +4,128 @@ using System.Text; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Rnn { - public class SimpleRNNCell : Layer + /// + /// Cell class for SimpleRNN. + /// See [the Keras RNN API guide](https://www.tensorflow.org/guide/keras/rnn) + /// for details about the usage of RNN API. + /// This class processes one step within the whole time sequence input, whereas + /// `tf.keras.layer.SimpleRNN` processes the whole sequence. + /// + public class SimpleRNNCell : DropoutRNNCellMixin { - SimpleRNNArgs args; - IVariableV1 kernel; - IVariableV1 recurrent_kernel; - IVariableV1 bias; + SimpleRNNCellArgs _args; + IVariableV1 _kernel; + IVariableV1 _recurrent_kernel; + IVariableV1 _bias; + GeneralizedTensorShape _state_size; + GeneralizedTensorShape _output_size; - public SimpleRNNCell(SimpleRNNArgs args) : base(args) + public override GeneralizedTensorShape StateSize => _state_size; + public override GeneralizedTensorShape OutputSize => _output_size; + public override bool SupportOptionalArgs => false; + + public SimpleRNNCell(SimpleRNNCellArgs args) : base(args) { - this.args = args; + this._args = args; + if (args.Units <= 0) + { + throw new ValueError( + $"units must be a positive integer, got {args.Units}"); + } + this._args.Dropout = Math.Min(1f, Math.Max(0f, this._args.Dropout)); + this._args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this._args.RecurrentDropout)); + _state_size = new GeneralizedTensorShape(args.Units); + _output_size = new GeneralizedTensorShape(args.Units); } public override void build(KerasShapesWrapper input_shape) { + // TODO(Rinne): add the cache. var single_shape = input_shape.ToSingleShape(); var input_dim = single_shape[-1]; - kernel = add_weight("kernel", (single_shape[-1], args.Units), - initializer: args.KernelInitializer + _kernel = add_weight("kernel", (single_shape[-1], _args.Units), + initializer: _args.KernelInitializer ); - recurrent_kernel = add_weight("recurrent_kernel", (args.Units, args.Units), - initializer: args.RecurrentInitializer + _recurrent_kernel = add_weight("recurrent_kernel", (_args.Units, _args.Units), + initializer: _args.RecurrentInitializer ); - if (args.UseBias) + if (_args.UseBias) { - bias = add_weight("bias", (args.Units), - initializer: args.BiasInitializer + _bias = add_weight("bias", (_args.Units), + initializer: _args.BiasInitializer ); } built = true; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + public override (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) { - return base.Call(inputs, state, training); + // TODO(Rinne): check if it will have multiple tensors when not nested. + Tensor prev_output = states[0]; + var dp_mask = get_dropout_maskcell_for_cell(inputs, training.Value); + var rec_dp_mask = get_recurrent_dropout_maskcell_for_cell(prev_output, training.Value); + + Tensor h; + var ranks = inputs.rank; + if (dp_mask != null) + { + if (ranks > 2) + { + // 因为multiply函数会自动添加第一个维度，所以加上下标0 + h = tf.linalg.tensordot(math_ops.multiply(inputs, dp_mask)[0], _kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + h = math_ops.matmul(math_ops.multiply(inputs, dp_mask)[0], _kernel.AsTensor()); + } + } + else + { + if (ranks > 2) + { + h = tf.linalg.tensordot(inputs, _kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + h = math_ops.matmul(inputs, _kernel.AsTensor()); + } + } + + if (_bias != null) + { + h = tf.nn.bias_add(h, _bias); + } + + if (rec_dp_mask != null) + { + prev_output = math_ops.multiply(prev_output, rec_dp_mask)[0]; + } + + ranks = prev_output.rank; + Tensor output; + if (ranks > 2) + { + output = h + tf.linalg.tensordot(prev_output[0], _recurrent_kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + } + else + { + output = h + math_ops.matmul(prev_output, _recurrent_kernel.AsTensor()); + } + Console.WriteLine($"shape of output: {output.shape}"); + + if (_args.Activation != null) + { + output = _args.Activation.Apply(output); + } + return (output, new Tensors { output }); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 20962df1..7923192f 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; @@ -8,7 +9,7 @@ using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Layers.Rnn { - public class StackedRNNCells : Layer, RNNArgs.IRnnArgCell + public class StackedRNNCells : Layer, IRnnCell { public IList Cells { get; set; } public bool reverse_state_order; @@ -51,7 +52,7 @@ namespace Tensorflow.Keras.Layers.Rnn { return lastCell.output_size; } - else if (RNN._is_multiple_state(lastCell.state_size)) + else if (RNN.is_multiple_state(lastCell.StateSize)) { // return ((dynamic)Cells[-1].state_size)[0]; throw new NotImplementedException(""); @@ -162,5 +163,13 @@ namespace Tensorflow.Keras.Layers.Rnn // deserialize_layer(cell_config, custom_objects = custom_objects)) // return cls(cells, **config) } + + public (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) + { + throw new NotImplementedException(); + } + public GeneralizedTensorShape StateSize => throw new NotImplementedException(); + public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); + public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs b/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs index 1ac4a277..6dfec319 100644 --- a/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs +++ b/src/TensorFlowNET.Keras/Layers/TensorFlowOpLayer.cs @@ -10,6 +10,7 @@ using Tensorflow.Keras.Engine; using static Tensorflow.Binding; using Tensorflow.Functions; using System.Threading; +using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { @@ -34,7 +35,7 @@ namespace Tensorflow.Keras.Layers built = true; } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { if (tf.Context.executing_eagerly()) return DeFunCall(inputs); diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index be6a49ec..3c2f8a7b 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -304,7 +304,7 @@ public class metrics_utils var NEG_INF = -1e10; var (_, top_k_idx) = tf.math.top_k(x, k, sorted: false); var top_k_mask = tf.reduce_sum( - tf.one_hot(top_k_idx, (int)x.shape[-1], axis: -1), axis: -2); + tf.one_hot(top_k_idx.Single, (int)x.shape[-1], axis: -1), axis: -2); return x * top_k_mask + NEG_INF * (1 - top_k_mask); } } diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs index fa19987b..4acae426 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs @@ -129,7 +129,7 @@ namespace Tensorflow.Keras var indices = z.map(m => { var (i, positions) = m; - return tf.range(positions[i], positions[i] + sequence_length_tensor * sampling_rate_tensor, sampling_rate_tensor); + return tf.range(positions.Single[i], positions.Single[i] + sequence_length_tensor * sampling_rate_tensor, sampling_rate_tensor); }, num_parallel_calls: -1); var dataset = sequences_from_indices(data, indices, start_index, end_index); diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index a26879e0..396ad20e 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -8,7 +8,7 @@ using System.Diagnostics; using System.Linq; using System.Reflection; using System.Text.RegularExpressions; -using Tensorflow.Extensions; +using Tensorflow.Common.Extensions; using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; diff --git a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs new file mode 100644 index 00000000..3109eb77 --- /dev/null +++ b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Common.Extensions; + +namespace Tensorflow.Keras.Utils +{ + internal static class RnnUtils + { + internal static Tensors generate_zero_filled_state(long batch_size_tensor, GeneralizedTensorShape state_size, TF_DataType dtype) + { + Func create_zeros; + create_zeros = (GeneralizedTensorShape unnested_state_size) => + { + var flat_dims = unnested_state_size.ToSingleShape().dims; + var init_state_size = new long[] { batch_size_tensor }.Concat(flat_dims).ToArray(); + return array_ops.zeros(new Shape(init_state_size), dtype: dtype); + }; + + // TODO(Rinne): map structure with nested tensors. + if(state_size.Shapes.Length > 1) + { + return new Tensors(state_size.ToShapeArray().Select(s => create_zeros(new GeneralizedTensorShape(s)))); + } + else + { + return create_zeros(state_size); + } + + } + + internal static Tensors generate_zero_filled_state_for_cell(IRnnCell cell, Tensors inputs, long batch_size, TF_DataType dtype) + { + if (inputs != null) + { + batch_size = inputs.shape[0]; + dtype = inputs.dtype; + } + return generate_zero_filled_state(batch_size, cell.StateSize, dtype); + } + + /// + /// Standardizes `__call__` to a single list of tensor inputs. + /// + /// When running a model loaded from a file, the input tensors + /// `initial_state` and `constants` can be passed to `RNN.__call__()` as part + /// of `inputs` instead of by the dedicated keyword arguments.This method + /// makes sure the arguments are separated and that `initial_state` and + /// `constants` are lists of tensors(or None). + /// + /// Tensor or list/tuple of tensors. which may include constants + /// and initial states.In that case `num_constant` must be specified. + /// Tensor or list of tensors or None, initial states. + /// Tensor or list of tensors or None, constant tensors. + /// Expected number of constants (if constants are passed as + /// part of the `inputs` list. + /// + internal static (Tensors, Tensors, Tensors) standardize_args(Tensors inputs, Tensors initial_state, Tensors constants, int num_constants) + { + if(inputs.Length > 1) + { + // There are several situations here: + // In the graph mode, __call__ will be only called once. The initial_state + // and constants could be in inputs (from file loading). + // In the eager mode, __call__ will be called twice, once during + // rnn_layer(inputs=input_t, constants=c_t, ...), and second time will be + // model.fit/train_on_batch/predict with real np data. In the second case, + // the inputs will contain initial_state and constants as eager tensor. + // + // For either case, the real input is the first item in the list, which + // could be a nested structure itself. Then followed by initial_states, which + // could be a list of items, or list of list if the initial_state is complex + // structure, and finally followed by constants which is a flat list. + Debug.Assert(initial_state is null && constants is null); + if(num_constants > 0) + { + constants = inputs.TakeLast(num_constants).ToTensors(); + inputs = inputs.SkipLast(num_constants).ToTensors(); + } + if(inputs.Length > 1) + { + initial_state = inputs.Skip(1).ToTensors(); + inputs = inputs.Take(1).ToTensors(); + } + } + + return (inputs, initial_state, constants); + } + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index 3de33746..f4980b82 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -144,17 +144,6 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual(expected_output, actual_output); } - [TestMethod, Ignore("WIP")] - public void SimpleRNN() - { - var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); - /*var simple_rnn = keras.layers.SimpleRNN(4); - var output = simple_rnn.Apply(inputs); - Assert.AreEqual((32, 4), output.shape);*/ - var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); - var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); - } - [TestMethod] public void Resizing() { diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs new file mode 100644 index 00000000..55663d41 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -0,0 +1,28 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tensorflow.NumPy; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras.UnitTest.Layers +{ + [TestClass] + public class Rnn + { + [TestMethod] + public void SimpleRNN() + { + var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); + /*var simple_rnn = keras.layers.SimpleRNN(4); + var output = simple_rnn.Apply(inputs); + Assert.AreEqual((32, 4), output.shape);*/ + var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); + var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); + Console.WriteLine(whole_sequence_output); + Console.WriteLine(final_state); + } + } +} diff --git a/tools/TensorFlowNET.Console/SimpleRnnTest.cs b/tools/TensorFlowNET.Console/SimpleRnnTest.cs index 9769eb65..ae6ebb8a 100644 --- a/tools/TensorFlowNET.Console/SimpleRnnTest.cs +++ b/tools/TensorFlowNET.Console/SimpleRnnTest.cs @@ -20,7 +20,7 @@ namespace Tensorflow // whole_sequence_output has shape `[32, 10, 4]`. // final_state has shape `[32, 4]`. - var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); + var (whole_sequence_output, final_states) = simple_rnn.Apply(inputs); } } } From 4939105b8f2de49d1f943c7edafd1b35690366ff Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Wed, 7 Jun 2023 07:56:19 +0800 Subject: [PATCH 03/77] feat: add Nestable and Nest. --- .../Common/Extensions/LinqExtensions.cs | 7 + .../Common/Extensions/NestExtensions.cs | 33 ++ .../Common/Types/GeneralizedTensorShape.cs | 53 +- src/TensorFlowNET.Core/Common/Types/INest.cs | 27 ++ .../Common/Types/INestable.cs | 11 + .../Common/Types/Nest.Static.cs | 62 +++ src/TensorFlowNET.Core/Common/Types/Nest.cs | 458 ++++++++++++++++++ .../Common/Types/NestDictionary.cs | 99 ++++ .../Common/Types/NestList.cs | 43 ++ .../Common/Types/NestNode.cs | 32 ++ src/TensorFlowNET.Core/Tensors/Tensors.cs | 211 ++++---- src/TensorFlowNET.Core/Util/nest.py.cs | 34 +- 12 files changed, 946 insertions(+), 124 deletions(-) create mode 100644 src/TensorFlowNET.Core/Common/Extensions/NestExtensions.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/INest.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/INestable.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/Nest.Static.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/Nest.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/NestDictionary.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/NestList.cs create mode 100644 src/TensorFlowNET.Core/Common/Types/NestNode.cs diff --git a/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs index 0402fca0..6cf62e7b 100644 --- a/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs +++ b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs @@ -22,5 +22,12 @@ namespace Tensorflow.Common.Extensions { return new Tensors(tensors); } + + public static void Deconstruct(this (T1, T2, T3) values, out T1 first, out T2 second, out T3 third) + { + first = values.Item1; + second = values.Item2; + third = values.Item3; + } } } diff --git a/src/TensorFlowNET.Core/Common/Extensions/NestExtensions.cs b/src/TensorFlowNET.Core/Common/Extensions/NestExtensions.cs new file mode 100644 index 00000000..76bdd613 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Extensions/NestExtensions.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; + +namespace Tensorflow.Common.Extensions +{ + public static class NestExtensions + { + public static Tensors ToTensors(this INestable tensors) + { + return new Tensors(tensors.AsNest()); + } + + public static Tensors? ToTensors(this Nest tensors) + { + return Tensors.FromNest(tensors); + } + + /// + /// If the nested object is already a nested type, this function could reduce it. + /// For example, `Nest[Nest[T]]` can be reduced to `Nest[T]`. + /// + /// + /// + /// + /// + public static Nest ReduceTo(this INestStructure input) where TIn: INestStructure + { + return Nest.ReduceFrom(input); + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs index edb9a802..e05d3deb 100644 --- a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs +++ b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs @@ -5,7 +5,7 @@ using System.Text; namespace Tensorflow.Common.Types { - public class GeneralizedTensorShape: IEnumerable + public class GeneralizedTensorShape: IEnumerable, INestStructure, INestable { public TensorShapeConfig[] Shapes { get; set; } /// @@ -63,6 +63,57 @@ namespace Tensorflow.Common.Types return Shapes.Select(x => new Shape(x.Items.Select(y => y is null ? -1 : y.Value).ToArray())).ToArray(); } + public IEnumerable Flatten() + { + List result = new List(); + foreach(var shapeConfig in Shapes) + { + result.AddRange(shapeConfig.Items); + } + return result; + } + public INestStructure MapStructure(Func func) + { + List> lists = new(); + foreach(var shapeConfig in Shapes) + { + lists.Add(new Nest(shapeConfig.Items.Select(x => new Nest(func(x))))); + } + return new Nest(lists); + } + + public Nest AsNest() + { + Nest DealWithSingleShape(TensorShapeConfig config) + { + if (config.Items.Length == 0) + { + return Nest.Empty; + } + else if (config.Items.Length == 1) + { + return new Nest(config.Items[0]); + } + else + { + return new Nest(config.Items.Select(x => new Nest(x))); + } + } + + if(Shapes.Length == 0) + { + return Nest.Empty; + } + else if(Shapes.Length == 1) + { + return DealWithSingleShape(Shapes[0]); + } + else + { + return new Nest(Shapes.Select(s => DealWithSingleShape(s))); + } + } + public IEnumerator GetEnumerator() { foreach (var shape in Shapes) diff --git a/src/TensorFlowNET.Core/Common/Types/INest.cs b/src/TensorFlowNET.Core/Common/Types/INest.cs new file mode 100644 index 00000000..001141dd --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/INest.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + /// + /// This interface indicates that a class may have a nested structure and provide + /// methods to manipulate with the structure. + /// + public interface INestStructure: INestable + { + /// + /// Flatten the Nestable object. Node that if the object contains only one value, + /// it will be flattened to an enumerable with one element. + /// + /// + IEnumerable Flatten(); + /// + /// Construct a new object with the same nested structure. + /// + /// + /// + /// + INestStructure MapStructure(Func func); + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/INestable.cs b/src/TensorFlowNET.Core/Common/Types/INestable.cs new file mode 100644 index 00000000..7ce49f85 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/INestable.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + public interface INestable + { + Nest AsNest(); + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs b/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs new file mode 100644 index 00000000..b67d11f4 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + public static class Nest + { + /// + /// Pack the flat items to a nested sequence by the template. + /// + /// + /// + /// + /// + public static Nest PackSequenceAs(INestable template, T[] flatItems) + { + return template.AsNest().PackSequence(flatItems); + } + + /// + /// Pack the flat items to a nested sequence by the template. + /// + /// + /// + /// + /// + public static Nest PackSequenceAs(INestable template, List flatItems) + { + return template.AsNest().PackSequence(flatItems.ToArray()); + } + + /// + /// Flatten the nested object. + /// + /// + /// + /// + public static IEnumerable Flatten(INestable nestedObject) + { + return nestedObject.AsNest().Flatten(); + } + + /// + /// Map the structure with specified function. + /// + /// + /// + /// + /// + /// + public static INestStructure MapStructure(Func func, INestable nestedObject) + { + return nestedObject.AsNest().MapStructure(func); + } + + public static bool IsNested(INestable obj) + { + return obj.AsNest().IsNested(); + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.cs b/src/TensorFlowNET.Core/Common/Types/Nest.cs new file mode 100644 index 00000000..84a60402 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/Nest.cs @@ -0,0 +1,458 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Extensions; + +namespace Tensorflow.Common.Types +{ + public enum NestType + { + Empty, + Node, + List, + Dictionary + } + + /// + /// A nested structure which may inclulde value, list and dictionary. + /// Note that dictionary does not ensure the data order. When using it as IEnumerable, + /// its order is depth-first. + /// + /// + public class Nest : INestStructure, IEnumerable + { + private static readonly Nest _empty = new Nest() + { + NestType = NestType.Empty, + }; + public static Nest Empty => _empty; + public NestType NestType { get; protected set; } + public string? Name { get; set; } + public T? Value { get; protected set; } + public List>? ListValue { get; protected set; } + public Dictionary>? DictValue { get; protected set; } + + protected Nest() { } + + public Nest(T value, string? name = null) + { + Value = value; + Name = name; + NestType = NestType.Node; + } + + public Nest(IEnumerable> values, string? name = null) + { + ListValue = values.ToList(); + Name = name; + NestType = NestType.List; + } + + public Nest(Dictionary> value, string? name = null) + { + DictValue = value; + Name = name; + NestType = NestType.Dictionary; + } + + public Nest(Nest other) + { + NestType = other.NestType; + Value = other.Value; + DictValue = other.DictValue; + ListValue = other.ListValue; + Name = other.Name; + } + + public virtual IEnumerable Flatten() + { + return FlattenInternal(this); + } + public virtual INestStructure MapStructure(Func func) + { + return MapStructureInternal(func); + } + + /// + /// Pack the flat items to a nested sequence by the template. + /// + /// + /// + public virtual Nest PackSequence(T[] flatItems) + { + if(flatItems.Length == 0) + { + return Nest.Empty; + } + int index = 0; + return PackSequenceInternal(this, flatItems, ref index); + } + + private static Nest PackSequenceInternal(Nest template, T[] flatItems, ref int index) + { + if(template.NestType == NestType.Node) + { + if(index >= flatItems.Length) + { + throw new InvalidArgumentError("The template and flat items are not matched."); + } + return new Nest(flatItems[index++]); + } + else if(template.NestType == NestType.List) + { + List> nestedObjects = new List>(); + for (int i = 0; i < template.ListValue!.Count; i++) + { + nestedObjects.Add(PackSequenceInternal(template.ListValue![i], flatItems, ref index)); + } + return new Nest(nestedObjects); + } + else if(template.NestType == NestType.Node) + { + Dictionary> dict = new Dictionary>(); + foreach(var (key, value) in template.DictValue!) + { + dict[key] = PackSequenceInternal(value, flatItems, ref index); + } + return new Nest(dict); + } + // Consider Empty as invalid type. + throw new InvalidArgumentError("When using `PackSequenceAs`, the template cannot contain empty node."); + } + + public virtual Nest AsNest() + { + return this; + } + + public virtual Nest MergeWith(Nest? other) + { + if(other is null || other == Nest.Empty) + { + return this; + } + if(this == Nest.Empty) + { + return other; + } + if(NestType == NestType.Node && other.NestType == NestType.Node) + { + return new Nest(new Nest[] { this, other }); + } + else if(NestType == NestType.List && other.NestType == NestType.List) + { + return new Nest(this.ListValue!.Concat(other.ListValue!)); + } + else if(NestType == NestType.Dictionary && other.NestType == NestType.Dictionary) + { + return new Nest(this.DictValue!.Concat(other.DictValue!).ToDictionary(x => x.Key, x => x.Value)); + } + else + { + return new Nest(new Nest[] { this, other }); + } + } + + /// + /// To see if the nested object is really nested. Despite being called `Nest`, sometimes it's actually not + /// nested. For example, [1, 2, 3] is not nested, while [1, [2, 3]] is nested. + /// + /// + public bool IsNested() + { + if(NestType is NestType.Empty or NestType.Node) + { + return false; + } + else if(NestType is NestType.List) + { + foreach(var item in ListValue!) + { + if(item.NestType is NestType.List or NestType.Dictionary) + { + return true; + } + } + return false; + } + else + { + foreach (var item in DictValue!.Values) + { + if (item.NestType is NestType.List or NestType.Dictionary) + { + return true; + } + } + return false; + } + } + + [Obsolete("The indexer of Tensors is not encouraged because it leads to unclear meanings.")] + public T this[int index] + { + get + { + bool success = FindInternal(this, index, out var result); + if (success) + { + return result; + } + else + { + throw new IndexOutOfRangeException(); + } + } + set + { + bool success = SetInternal(this, index, value); + if (!success) + { + throw new IndexOutOfRangeException(); + } + } + } + + /// + /// If the existing nested structure if of type `Nest[INestStructure[T]]`, we can reduce it + /// to `Nest[T]`. + /// + /// + /// + /// + public static Nest ReduceFrom(INestStructure input) where TOut: INestStructure + { + var nested = input.AsNest(); + return ReduceInternal(nested); + } + + private static Nest ReduceInternal(Nest node) where TOut : INestStructure + { + if(node.NestType == NestType.Empty) + { + return Nest.Empty; + } + else if(node.NestType == NestType.Node) + { + return node.Value!.AsNest(); + } + else if(node.NestType == NestType.List) + { + return new Nest(node.ListValue!.Select(x => ReduceInternal(x))); + } + else // Dictionary type + { + return new Nest(node.DictValue!.ToDictionary(x => x.Key, x => ReduceInternal(x.Value))); + } + } + + private static bool FindInternal(Nest node, int index, out T? result) + { + if (node.NestType == NestType.Node) + { + if(index == 0) + { + result = node.Value!; + return true; + } + result = default(T); + return false; + } + else if (node.NestType == NestType.List) + { + foreach (var item in node.ListValue!) + { + if(index == 0) + { + return FindInternal(item, index, out result); + } + index--; + } + result = default(T); + return false; + } + else if(node.NestType == NestType.Dictionary) + { + foreach (var item in node.DictValue!.Values) + { + if (index == 0) + { + return FindInternal(item, index, out result); + } + index--; + } + result = default(T); + return false; + } + else + { + result = default(T); + return false; + } + } + + private static bool SetInternal(Nest node, int index, T newValue) + { + if (node.NestType == NestType.Node) + { + if (index == 0) + { + node.Value = newValue; + return true; + } + return false; + } + else if (node.NestType == NestType.List) + { + foreach (var item in node.ListValue!) + { + if (index == 0) + { + return SetInternal(item, index, newValue); + } + index--; + } + return false; + } + else if (node.NestType == NestType.Dictionary) + { + foreach (var item in node.DictValue!.Values) + { + if (index == 0) + { + return SetInternal(item, index, newValue); + } + index--; + } + return false; + } + else + { + return false; + } + } + + private static IEnumerable FlattenInternal(Nest node) + { + if (node.NestType == NestType.Node) + { + yield return node.Value!; + } + else if (node.NestType == NestType.List) + { + foreach (var item in node.ListValue!) + { + foreach(var val in FlattenInternal(item)) + { + yield return val; + } + } + } + else if (node.NestType == NestType.Dictionary) + { + foreach (var item in node.DictValue!.Values) + { + foreach (var val in FlattenInternal(item)) + { + yield return val; + } + } + } + } + + private Nest MapStructureInternal(Func func) + { + if (NestType == NestType.Node) + { + return new Nest(func(Value!)); + } + else if (NestType == NestType.List) + { + List> outs = new List>(); + foreach (var item in ListValue!) + { + outs.Add(item.MapStructureInternal(func)); + } + return new Nest(outs); + } + else if (NestType == NestType.Dictionary) + { + Dictionary> outs = new Dictionary>(); + foreach (var (key, value) in DictValue!) + { + outs.Add(key, value.MapStructureInternal(func)); + } + return new Nest(outs); + } + else + { + return Nest.Empty; + } + } + + public IEnumerator GetEnumerator() + { + return Flatten().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + sb.Append("("); + WriteString(this, sb); + sb.Append(")"); + return sb.ToString(); + } + + private static void WriteString(Nest node, StringBuilder sb) + { + if (!string.IsNullOrEmpty(node.Name)) + { + sb.Append($"{node.Name}: "); + } + if (node.NestType == NestType.Node) + { + sb.Append(node.Value!.ToString()); + } + else if (node.NestType == NestType.List) + { + sb.Append("["); + for(int i = 0; i < node.ListValue!.Count; i++) + { + WriteString(node.ListValue![i], sb); + if(i != node.ListValue!.Count - 1) + { + sb.Append(", "); + } + } + sb.Append("]"); + } + else if (node.NestType == NestType.Dictionary) + { + sb.Append("{"); + int count = node.DictValue!.Count; + int i = 0; + foreach (var (key, value) in node.DictValue!) + { + sb.Append($"{key}: "); + WriteString(value, sb); + if (i != count - 1) + { + sb.Append(", "); + } + i++; + } + sb.Append("}"); + } + else + { + sb.Append(""); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs b/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs new file mode 100644 index 00000000..554ca526 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + public class NestDictionary : INestStructure, IDictionary where TKey : notnull + { + public IDictionary Value { get; set; } + public NestDictionary(IDictionary dict) + { + Value = dict; + } + public IEnumerable Flatten() + { + return Value.Select(x => x.Value); + } + public INestStructure MapStructure(Func func) + { + return new NestList(Value.Select(x => func(x.Value))); + } + + public Nest AsNest() + { + return new Nest(Value.Values.Select(x => new Nest(x))); + } + + // Required IDictionary members + public int Count => Value.Count; + + public bool IsReadOnly => Value.IsReadOnly; + + public ICollection Keys => Value.Keys; + + public ICollection Values => Value.Values; + + public void Add(TKey key, TValue value) + { + Value.Add(key, value); + } + + public void Add(KeyValuePair item) + { + Value.Add(item); + } + + public void Clear() + { + Value.Clear(); + } + + public bool Contains(KeyValuePair item) + { + return Value.Contains(item); + } + + public bool ContainsKey(TKey key) + { + return Value.ContainsKey(key); + } + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + Value.CopyTo(array, arrayIndex); + } + + public IEnumerator> GetEnumerator() + { + return Value.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + public bool Remove(TKey key) + { + return Value.Remove(key); + } + + public bool Remove(KeyValuePair item) + { + return Value.Remove(item); + } + + public bool TryGetValue(TKey key, out TValue value) + { + return Value.TryGetValue(key, out value); + } + + // Optional IDictionary members + public TValue this[TKey key] + { + get => Value[key]; + set => Value[key] = value; + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/NestList.cs b/src/TensorFlowNET.Core/Common/Types/NestList.cs new file mode 100644 index 00000000..08218718 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/NestList.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + /// + /// The implementation of a list that support nest structure, in which the depth is 1. + /// + /// + public sealed class NestList : INestStructure, IEnumerable + { + public List Value { get; set; } + public NestList(IEnumerable values) + { + Value = new List(values); + } + public IEnumerable Flatten() + { + return Value; + } + public INestStructure MapStructure(Func func) + { + return new NestList(Value.Select(x => func(x))); + } + + public Nest AsNest() + { + return new Nest(Value.Select(x => new Nest(x))); + } + + // Enumerator implementation + public IEnumerator GetEnumerator() + { + return Value.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/NestNode.cs b/src/TensorFlowNET.Core/Common/Types/NestNode.cs new file mode 100644 index 00000000..1dad421d --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/NestNode.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + /// + /// A nested structure with only one element. + /// + /// + public class NestNode : INestStructure + { + public T Value { get; set; } + public NestNode(T value) + { + Value = value; + } + public IEnumerable Flatten() + { + yield return Value; + } + public INestStructure MapStructure(Func func) + { + return new NestNode(func(Value)); + } + + public Nest AsNest() + { + return new Nest(Value); + } + } +} diff --git a/src/TensorFlowNET.Core/Tensors/Tensors.cs b/src/TensorFlowNET.Core/Tensors/Tensors.cs index caa36b76..cba8f954 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensors.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensors.cs @@ -3,6 +3,7 @@ using System; using System.Collections; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; namespace Tensorflow { @@ -13,16 +14,14 @@ namespace Tensorflow /// and Tensor[] from Tensors implicitily. /// It works for tuple and scalar as well. /// - public class Tensors : IEnumerable, IDisposable + public sealed class Tensors : Nest, IDisposable { - List items = new List(); - - public TF_DataType dtype => items.First().dtype; - public Shape shape => items.First().shape; - public int rank => items.First().rank; - public Graph graph => items.First().graph; + public TF_DataType dtype => this.First().dtype; + public Shape shape => this.First().shape; + public int rank => this.First().rank; + public Graph graph => this.First().graph; public bool IsList { get; set; } - public int Length => items.Count(); + public int Length => this.Count(); /// /// Return a Tensor if `Tensors` has only one tensor, otherwise throw an exception. /// @@ -35,7 +34,7 @@ namespace Tensorflow throw new ValueError("Tensors with more than one tensor cannot be " + "implicitly converted to Tensor."); } - return items.First(); + return this.First(); } } @@ -52,150 +51,194 @@ namespace Tensorflow throw new ValueError($"Tensors with {Length} tensor cannot be " + "implicitly converted to Tensor."); } - return items.FirstOrDefault(); + return this.FirstOrDefault(); } } - public Tensor this[int index] + public Tensor this[params string[] slices] + => this.First()[slices]; + + public Tensors(Tensor tensor) : base(tensor) { - get => items[index]; - set => items[index] = value; + } - public Tensor this[params string[] slices] - => items.First()[slices]; - public Tensors(params Tensor[] tensors) + private Tensors(Nest nested) : base(nested) { - items.AddRange(tensors); + } - public Tensors(IEnumerable tensors) + public Tensors(params Tensor[] tensors): base(tensors.Select(x => new Nest(x))) { - items.AddRange(tensors); + } - public Tensors(NDArray nd) + public Tensors(IEnumerable tensors): base(tensors.Select(x => new Nest(x))) { - items.Add(ops.convert_to_tensor(nd)); + } - public IEnumerator GetEnumerator() + public Tensors(NDArray nd): base(ops.convert_to_tensor(nd)) { - foreach (var tensor in items) - yield return tensor; + } + public bool IsSingle() + { + return Length == 1; + } + + public new Tensors MergeWith(Nest? other) + { + return FromNest(base.MergeWith(other)); + } + + [Obsolete("This method is not encouraged to be used. It may be removed in the future. If you do want to add " + + "a tensor to `Tensors`, creating a new instance with your newly added tensor is a better choice.")] public void Add(Tensor tensor) - => items.Add(tensor); + { + if(NestType == NestType.Dictionary) + { + throw new ValueError("Cannot add a tensor to dictionary type of nested tensors."); + } + else if(NestType == NestType.Node) + { + NestType = NestType.List; + ListValue = new() { new Nest(Value), new Nest(tensor) }; + Value = null; + } + else + { + ListValue.Add(new Nest(tensor)); + } + } + [Obsolete("This method is not encouraged to be used. It may be removed in the future. If you do want to add " + + "some tensors to `Tensors`, creating a new instance with your newly added tensors is a better choice.")] public void AddRange(IEnumerable tensors) - => items.AddRange(tensors); + { + if (NestType == NestType.Dictionary) + { + throw new ValueError("Cannot add a tensor to dictionary type of nested tensors."); + } + else if (NestType == NestType.Node) + { + NestType = NestType.List; + ListValue = new() { new Nest(Value) }; + ListValue.AddRange(tensors.Select(x => new Nest(x))); + Value = null; + } + else + { + ListValue.AddRange(tensors.Select(x => new Nest(x))); + } + } + [Obsolete("This method is not encouraged to be used. It may be removed in the future. If you do want to insert " + + "a tensor to `Tensors`, creating a new instance with your newly added tensor is a better choice.")] public void Insert(int index, Tensor tensor) - => items.Insert(index, tensor); - - IEnumerator IEnumerable.GetEnumerator() - => GetEnumerator(); + { + if (NestType == NestType.List) + { + ListValue.Insert(index, new Nest(tensor)); + } + else if(NestType == NestType.Node) + { + NestType = NestType.List; + ListValue = new() { new Nest(Value) }; + ListValue.Insert(index, new Nest(tensor)); + Value = null; + } + else + { + throw new ValueError("Cannot add a tensor to dictionary type of nested tensors."); + } + } public string[] StringData() { - EnsureSingleTensor(this, "nnumpy"); - return this[0].StringData(); + return Single.StringData(); } public string StringData(int index) { - EnsureSingleTensor(this, "nnumpy"); - return this[0].StringData(index); + return Single.StringData(index); } public NDArray numpy() { - EnsureSingleTensor(this, "nnumpy"); - return this[0].numpy(); + return Single.numpy(); } + [Obsolete] public T[] ToArray() where T: unmanaged { - EnsureSingleTensor(this, $"ToArray<{typeof(T)}>"); - return this[0].ToArray(); + return Single.ToArray(); } #region Explicit Conversions public unsafe static explicit operator bool(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to bool"); - return (bool)tensor[0]; + return (bool)tensor.Single; } public unsafe static explicit operator sbyte(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to sbyte"); - return (sbyte)tensor[0]; + return (sbyte)tensor.Single; } public unsafe static explicit operator byte(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to byte"); - return (byte)tensor[0]; + return (byte)tensor.Single; } public unsafe static explicit operator ushort(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to ushort"); - return (ushort)tensor[0]; + return (ushort)tensor.Single; } public unsafe static explicit operator short(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to short"); - return (short)tensor[0]; + return (short)tensor.Single; } public unsafe static explicit operator int(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to int"); - return (int)tensor[0]; + return (int)tensor.Single; } public unsafe static explicit operator uint(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to uint"); - return (uint)tensor[0]; + return (uint)tensor.Single; } public unsafe static explicit operator long(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to long"); - return (long)tensor[0]; + return (long)tensor.Single; } public unsafe static explicit operator ulong(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to ulong"); - return (ulong)tensor[0]; + return (ulong)tensor.Single; } public unsafe static explicit operator float(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to byte"); - return (byte)tensor[0]; + return (byte)tensor.Single; } public unsafe static explicit operator double(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to double"); - return (double)tensor[0]; + return (double)tensor.Single; } public unsafe static explicit operator string(Tensors tensor) { - EnsureSingleTensor(tensor, "explicit conversion to string"); - return (string)tensor[0]; + return (string)tensor.Single; } public static explicit operator object[](Tensors tensors) - => tensors.items.ToArray(); + => tensors.Flatten().ToArray(); #endregion #region Implicit Conversions @@ -219,52 +262,40 @@ namespace Tensorflow => tensors?.SingleOrNull; public static implicit operator Tensor[](Tensors tensors) - => tensors.items.ToArray(); - + => tensors.Flatten().ToArray(); #endregion - public void Deconstruct(out Tensor a, out Tensors? b) + public static Tensors? FromNest(Nest nested) { - a = items[0]; - b = Length == 1? null : new Tensors(items.Skip(1)); + if(nested == Nest.Empty) + { + return null; + } + return new Tensors(nested); } - private static void EnsureSingleTensor(Tensors tensors, string methodnName) + public void Deconstruct(out Tensor a, out Tensors? b) { - if(tensors.Length == 0) - { - throw new ValueError($"Method `{methodnName}` of `Tensors` cannot be used when `Tensors` contains no Tensor."); - } - else if(tensors.Length > 1) - { - throw new ValueError($"Method `{methodnName}` of `Tensors` cannot be used when `Tensors` contains more than one Tensor."); - } + a = this.First(); + b = Length == 1? null : new Tensors(this.Skip(1)); } public override string ToString() { - if(items.Count == 1) + if(Length == 1) { - return items[0].ToString(); + return this.First().ToString(); } else { - StringBuilder sb = new StringBuilder(); - sb.Append($"Totally {items.Count} tensors, which are {string.Join(", ", items.Select(x => x.name))}\n[\n"); - for(int i = 0; i < items.Count; i++) - { - var tensor = items[i]; - sb.Append($"Tensor {i}({tensor.name}): {tensor.ToString()}\n"); - } - sb.Append("]\n"); - return sb.ToString(); + return $"Totally {Length} tensors: {base.ToString()}"; } } public void Dispose() { - foreach (var item in items) - item.Dispose(); + foreach (var tensor in this) + tensor.Dispose(); } } } diff --git a/src/TensorFlowNET.Core/Util/nest.py.cs b/src/TensorFlowNET.Core/Util/nest.py.cs index ab6f56b3..3ba3ce78 100644 --- a/src/TensorFlowNET.Core/Util/nest.py.cs +++ b/src/TensorFlowNET.Core/Util/nest.py.cs @@ -36,6 +36,7 @@ namespace Tensorflow.Util // (np.array([3, 4]), tf.constant([3, 4])))` // + [Obsolete] public static class nest { @@ -170,39 +171,6 @@ namespace Tensorflow.Util throw new TypeError("Type of sequence not supported (yet): " + instance.GetType()); } - public static bool is_nested(object obj) - { - // Refer to https://www.tensorflow.org/api_docs/python/tf/nest - //if (obj is IList || obj is IDictionary || obj is ITuple) - // return true; - if (obj is IList || obj is IDictionary) - return true; - - if (obj is NDArray || obj is Tensor || obj is string || obj.GetType().IsGenericType - || obj is ISet || obj is ISet || obj is ISet) - return false; - - if (obj.GetType().IsNested) return true; - // Check if the object is an IEnumerable - if (obj is IEnumerable) - { - // If it is, check if it is a nested structure - foreach (object item in (IEnumerable)obj) - { - if (is_nested(item)) - { - return true; - } - } - return true; - } - else - { - // If it is not, return false - return false; - } - } - /// /// Yields the next value from the given iterable. /// From 537b3e11428db323a1e9bf59e686fdf8c08e8eeb Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Wed, 7 Jun 2023 07:56:49 +0800 Subject: [PATCH 04/77] feat: support simple RNN. --- .../Keras/Layers/Rnn/IRnnCell.cs | 2 +- .../Operations/NnOps/RNNCell.cs | 1 + .../Operations/_EagerTensorArray.cs | 117 ++- src/TensorFlowNET.Keras/BackendImpl.cs | 721 +++++++++--------- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 24 +- .../Layers/Rnn/RnnCellBase.cs | 2 +- .../Layers/Rnn/SimpleRNNCell.cs | 50 +- .../Layers/Rnn/StackedRNNCells.cs | 1 + src/TensorflowNET.Hub/KerasLayer.cs | 3 +- 9 files changed, 507 insertions(+), 414 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs index df6222cd..d12ed1ad 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -9,11 +9,11 @@ namespace Tensorflow.Keras.Layers.Rnn { GeneralizedTensorShape StateSize { get; } GeneralizedTensorShape OutputSize { get; } + bool IsTFRnnCell { get; } /// /// Whether the optional RNN args are supported when appying the layer. /// In other words, whether `Apply` is overwrited with process of `RnnOptionalArgs`. /// bool SupportOptionalArgs { get; } - (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null); } } diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 71fdc301..26646b76 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -183,6 +183,7 @@ namespace Tensorflow } public GeneralizedTensorShape StateSize => throw new NotImplementedException(); public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); + public bool IsTFRnnCell => throw new NotImplementedException(); public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs index cf1b50af..ed65a08d 100644 --- a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Eager; using Tensorflow.Framework; using static Tensorflow.Binding; @@ -48,6 +49,7 @@ namespace Tensorflow.Operations public override Tensor flow => _flow; bool _clear_after_read; List _tensor_array; + List _previous_read_indices; public _EagerTensorArray(TF_DataType dtype, Tensor size, bool dynamic_size = false, bool clear_after_read = true, string tensor_array_name = null, Tensor handle = null, Tensor flow = null, @@ -61,16 +63,20 @@ namespace Tensorflow.Operations _dtype = dtype.as_base_dtype(); _dynamic_size = dynamic_size; _clear_after_read = clear_after_read; - _tensor_array = new List(); + _tensor_array = Enumerable.Repeat(null, size.numpy()).ToList(); + _previous_read_indices = new(); } public override TensorArray unstack(Tensor value, string name = null) { - return tf_with(ops.name_scope(name, "TensorArrayUnstack", new { _handle, value }), delegate + var tensors = array_ops.unstack(value, name: name); + if(tensors.Length > _tensor_array.Count && !_dynamic_size) { - var num_elements = array_ops.shape(value)[0]; - return scatter(indices: math_ops.range(0, num_elements), value: value, name: name); - }); + throw new ValueError($"Cannot unstack {tensors.Length} tensors into a TensorArray of static size {_tensor_array.Count}"); + } + _tensor_array = tensors.ToList(); + // TODO(Rinne): revise the implementation. Here we should return `parent()`. + return this; } public TensorArray scatter(Tensor indices, Tensor value, string name = null) @@ -116,9 +122,19 @@ namespace Tensorflow.Operations _colocate_with.Add(value); } + private Tensor _maybe_zero(int ix) + { + var val = _tensor_array[ix]; + if(val is null) + { + val = _tensor_array[ix] = array_ops.zeros(_element_shape, _dtype); + } + return val; + } + public override Tensor read(T index, string name = null) { - int index_int = -1; + int index_int; if (index is int int_index) index_int = int_index; else if (index is Tensor tensor_index) @@ -126,27 +142,75 @@ namespace Tensorflow.Operations else throw new ValueError(""); + if(index_int >= _tensor_array.Count) + { + throw new OutOfRangeError($"Tried to read from index {index_int} but array size is: {_tensor_array.Count} "); + } + + var res = _tensor_array[index_int]; + if(res is null) + { + if (_previous_read_indices.Contains(index_int)) + { + throw new InvalidArgumentError($"Could not read index {index_int} twice because it was cleared after " + + $"a previous read (perhaps try setting clear_after_read = false?)"); + } + else + { + res = _maybe_zero(index_int); + } + } + if (_clear_after_read) { _tensor_array[index_int] = null; + _previous_read_indices.Add(index_int); } - - return _tensor_array[index_int]; + return res; } public override TensorArray write(Tensor index, Tensor value, string name = null) { - if (_infer_shape) - _element_shape = _element_shape.merge_with(value.shape); - _tensor_array.add(value); - return this; + int index_int; + if(index is EagerTensor eager) + { + return write(eager.numpy(), value, name); + } + throw new InvalidArgumentError("The index is supposed to be an EagerTensor"); } public override TensorArray write(int index, T value, string name = null) { - var value_tensor = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); - var index_tensor = ops.convert_to_tensor(index, name: "index"); - return write(index_tensor, value_tensor, name: name); + int size = _tensor_array.Count; + if(index >= size) + { + if (!_dynamic_size) + { + throw new OutOfRangeError($"Tried to write to index {index} but array is not resizeable and size " + + $"is: {size} "); + } + _tensor_array.AddRange(Enumerable.Repeat(null, index - size + 1)); + } + + Tensor tensor = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); + + if(_dtype != tensor.dtype) + { + throw new InvalidArgumentError($"TensorArray dtype is {_dtype.as_python_name()} but Op is " + + $"trying to write dtype {tensor.dtype.as_python_name()} "); + } + + if (!_element_shape.is_compatible_with(tensor.shape)) + { + throw new ValueError($"Incompatible shape for value ({tensor.shape}), expected ({_element_shape})"); + } + + if (_infer_shape) + { + _element_shape = _element_shape.merge_with(tensor.shape); + } + _tensor_array[index] = tensor; + return this; } private Tensor size(string name = null) @@ -156,11 +220,26 @@ namespace Tensorflow.Operations public override Tensor stack(string name = null) { - ops.colocate_with(_handle); - return tf_with(ops.name_scope(name, "TensorArrayStack", new { _handle }), delegate + if(_tensor_array.Count > 0) { - return gather(math_ops.range(0, size()), name: name); - }); + for(int i = 0; i < _tensor_array.Count; i++) + { + _maybe_zero(i); + } + } + if(_tensor_array.Count == 0 && _element_shape.IsFullyDefined) + { + return ops.convert_to_tensor(new Shape(new long[] { 0 }.Concat(_element_shape.dims).ToArray()), name: name, dtype: _dtype); + } + else + { + return ops.convert_to_tensor(_tensor_array, name: name, dtype: _dtype); + } + //ops.colocate_with(_handle); + //return tf_with(ops.name_scope(name, "TensorArrayStack", new { _handle }), delegate + //{ + // return gather(math_ops.range(0, size()), name: name); + //}); } public override Tensor gather(Tensor indices, string name = null) diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index a7c1bcad..30b73e82 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -20,9 +20,11 @@ using System.Linq; using System.Collections.Generic; using Tensorflow.Functions; using Tensorflow.Graphs; +using Tensorflow.Common.Extensions; using static Tensorflow.Binding; using static Tensorflow.Graphs.SubGraphUtility; using Tensorflow.Util; +using Tensorflow.Common.Types; namespace Tensorflow.Keras { @@ -452,7 +454,7 @@ namespace Tensorflow.Keras return x; } - public static (Tensors, Tensors, Tensors) rnn( + public (Tensors, Tensors, Tensors) rnn( Func step_function, // args:inputs, states, return:output, new_states Tensors inputs, // inputs is a tuple of tensors (one per input sequence) Tensors initial_states, @@ -466,7 +468,7 @@ namespace Tensorflow.Keras bool return_all_outputs = true) { - Tensors swap_batch_timestep(Tensors input_t) + Tensor swap_batch_timestep(Tensor input_t) { var axes = Enumerable.Range(0, input_t.rank).ToArray(); axes[0] = 1; @@ -476,13 +478,14 @@ namespace Tensorflow.Keras if (!time_major) { - inputs = nest.map_structure(swap_batch_timestep, inputs); + inputs = Nest.MapStructure(swap_batch_timestep, inputs).ToTensors(); } - var flatted_inptus = nest.flatten(inputs); - var time_steps = flatted_inptus[0].shape[0]; - var batch = flatted_inptus[0].shape[1]; - var time_step_t = tf.shape(flatted_inptus[0])[0]; + var flatted_inptus = Nest.Flatten(inputs).ToList(); + var first_flatted_input = flatted_inptus[0]; + var time_steps = first_flatted_input.shape[0]; + var batch = first_flatted_input.shape[1]; + var time_steps_t = (int)first_flatted_input.shape[0]; foreach (var input_ in flatted_inptus) { @@ -508,11 +511,6 @@ namespace Tensorflow.Keras } - if (constants == null) - { - constants = new List(); - } - // tf.where needs its condition tensor to be the same shape as its two // result tensors, but in our case the condition (mask) tensor is // (nsamples, 1), and inputs are (nsamples, ndimensions) or even more. @@ -522,12 +520,12 @@ namespace Tensorflow.Keras Tensors _expand_mask(Tensors mask_t, Tensors input_t, int fixed_dim = 1) { - if (nest.is_nested(mask_t)) + if (!mask_t.IsSingle()) { throw new ValueError($"mask_t is expected to be tensor, but got {mask_t}"); } - if (nest.is_nested(input_t)) + if (!input_t.IsSingle()) { throw new ValueError($"input_t is expected to be tensor, but got {input_t}"); } @@ -575,21 +573,21 @@ namespace Tensorflow.Keras - Tensors _process_single_input_t(Tensors input_t) + Tensors _process_single_input_t(Tensor input_t) { - input_t = tf.unstack(input_t); // unstack for time_step dim + var unstaked_input_t = array_ops.unstack(input_t); // unstack for time_step dim if (go_backwards) { - input_t.Reverse(); + unstaked_input_t = unstaked_input_t.Reverse().ToArray(); } - return input_t; + return unstaked_input_t; } // TODO(Wanglongzhi2001) Tensors processed_input; - if (nest.is_nested(inputs)) + if (!inputs.IsSingle()) { - processed_input = nest.map_structure(_process_single_input_t, inputs); + processed_input = inputs.MapStructure(_process_single_input_t).ReduceTo().ToTensors(); } else { @@ -603,334 +601,339 @@ namespace Tensorflow.Keras { inp.Add(t_[time]); } - return nest.pack_sequence_as(inputs, inp); + return Nest.PackSequenceAs(inputs, inp); } - //if (mask != null) - //{ - // var mask_list = tf.unstack(mask); - // if (go_backwards) - // { - // mask_list.Reverse(); - // } - - // for (int i = 0; i < time_steps; i++) - // { - // // TODO(Wanglongzhi2001),deal with _get_input_tensor - // var inp = _get_input_tensor(i); - // var mask_t = mask_list[i]; - // // TODO - // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); - - // var tiled_mask_t = _expand_mask(mask_t, output); - - // Tensors prev_output; - // if (successive_outputs == null) - // { - // prev_output = tf.zeros_like(output); - // } - // else - // { - // prev_output = successive_outputs[successive_outputs.Length - 1]; - // } - - // output = tf.where(tiled_mask_t, output, prev_output); - - // //var flat_states = nest.flatten(states); - // //var flat_new_states = nest.flatten(newStates); - // var flat_states = states.ToList(); - // var flat_new_states = newStates.ToList(); - - // var tiledMaskT = flat_states - // .Select(s => _expand_mask(mask_t, s)) - // .ToArray(); - // var tuple = Tuple.Create(tiledMaskT); - - // List flat_final_states = new List(); - // foreach (var (m, s, ps) in Enumerable.Zip(tiled_mask_t, flat_new_states, flat_states)) - // { - // flat_final_states.Add(tf.where(m, s, ps)); - // } - - // states = (Tensors)nest.pack_sequence_as(states, flat_final_states); - // if (return_all_outputs) - // { - // successive_outputs.Add(output); - // successive_states.Add(states); - // } - // else - // { - // successive_outputs = new Tensors { output }; - // successive_states = new Tensors { states }; - // } - - // } - // last_output = successive_outputs[successive_outputs.Length - 1]; - // new_states = successive_states[successive_states.Length - 1]; - // outputs = tf.stack(successive_outputs); - - // if (zero_output_for_mask) - // { - // last_output = tf.where(_expand_mask(mask_list[mask_list.Length - 1], last_output), last_output, tf.zeros_like(last_output)); - // outputs = tf.where(_expand_mask(mask, outputs, fixed_dim: 2), outputs, tf.zeros_like(outputs)); - // } - // else // mask is null - // { - // for (int i = 0; i < time_steps; i++) - // { - // var inp = _get_input_tensor(i); - // var (output, newStates) = step_function((Tensors)inp, new Tensors { states, constants }); - // states = newStates; - - // if (return_all_outputs) - // { - // successive_outputs.Add(output); - // successive_states.Add(newStates); - // } - // else - // { - // successive_outputs = new Tensors { output }; - // successive_states = new Tensors { newStates }; - // } - // } - // last_output = successive_outputs[successive_outputs.Length - 1]; - // new_states = successive_states[successive_states.Length - 1]; - // outputs = tf.stack(successive_outputs); - // } - //} + if (mask != null) + { + var mask_list = tf.unstack(mask); + if (go_backwards) + { + mask_list.Reverse(); + } + + for (int i = 0; i < time_steps; i++) + { + // TODO(Wanglongzhi2001),deal with _get_input_tensor + var inp = _get_input_tensor(i); + var mask_t = mask_list[i]; + // TODO + var (output, newStates) = step_function((Tensors)inp, states.MergeWith(constants)); + + var tiled_mask_t = _expand_mask(mask_t, output); + + Tensors prev_output; + if (successive_outputs == null) + { + prev_output = tf.zeros_like(output); + } + else + { + prev_output = successive_outputs[successive_outputs.Length - 1]; + } + + output = tf.where(tiled_mask_t, output, prev_output); + + var flat_states = Nest.Flatten(states).ToList(); + var flat_new_states = Nest.Flatten(newStates).ToList(); + + var tiledMaskT = flat_states + .Select(s => _expand_mask(mask_t, s)) + .ToArray(); + var tuple = Tuple.Create(tiledMaskT); + + List flat_final_states = new List(); + foreach (var (m, s, ps) in zip(tiled_mask_t.ToList(), flat_new_states, flat_states)) + { + flat_final_states.Add(tf.where(m, s, ps)); + } + + states = Nest.PackSequenceAs(states, flat_final_states).ToTensors(); + if (return_all_outputs) + { + successive_outputs.Add(output); + successive_states.Add(states); + } + else + { + successive_outputs = new Tensors { output }; + successive_states = new Tensors { states }; + } + + } + last_output = successive_outputs[successive_outputs.Length - 1]; + new_states = successive_states[successive_states.Length - 1]; + outputs = tf.stack(successive_outputs); + + if (zero_output_for_mask) + { + last_output = tf.where(_expand_mask(mask_list[mask_list.Length - 1], last_output), last_output, tf.zeros_like(last_output)); + outputs = tf.where(_expand_mask(mask, outputs, fixed_dim: 2), outputs, tf.zeros_like(outputs)); + } + else // mask is null + { + for (int i = 0; i < time_steps; i++) + { + var inp = _get_input_tensor(i); + var (output, newStates) = step_function((Tensors)inp, states.MergeWith(constants)); + states = newStates; + + if (return_all_outputs) + { + successive_outputs.Add(output); + successive_states.Add(newStates); + } + else + { + successive_outputs = new Tensors { output }; + successive_states = new Tensors { newStates }; + } + } + last_output = successive_outputs[successive_outputs.Length - 1]; + new_states = successive_states[successive_states.Length - 1]; + outputs = tf.stack(successive_outputs); + } + } + } + else // unroll == false + { + var states = initial_states; + // Create input tensor array, if the inputs is nested tensors, then it + // will be flattened first, and tensor array will be created one per + // flattened tensor. + var input_ta = new List(); + for (int i = 0; i < flatted_inptus.Count; i++) + { + input_ta.Add(tf.TensorArray(dtype: flatted_inptus[i].dtype, size: time_steps_t)); + } + + foreach(var (ta, input_) in zip(input_ta, flatted_inptus)) + { + if (!go_backwards) + { + ta.unstack(input_); + } + else + { + ta.unstack(reverse(input_, 0)); + } + } + + // Get the time(0) input and compute the output for that, the output will + // be used to determine the dtype of output tensor array. Don't read from + // input_ta due to TensorArray clear_after_read default to True. + var inps = new Tensors(); + foreach (var inp in flatted_inptus) + { + inps.Add(inp[0]); + } + var input_time_zero = Nest.PackSequenceAs(inputs, inps).ToTensors(); + + // output_time_zero is used to determine the cell output shape and its + // dtype. the value is discarded. + (output_time_zero, _) = step_function((Tensor)input_time_zero, + constants is null ? initial_states : initial_states.MergeWith(constants)); + + int output_ta_size = return_all_outputs ? time_steps_t : 1; + var output_ta = new List(); + for (int i = 0; i < output_time_zero.ToList().Count; i++) + { + var Out = output_time_zero.ToList()[i]; + output_ta.Add(tf.TensorArray(dtype: Out.dtype, size: output_ta_size, element_shape: Out.shape)); + } + + var time = tf.constant(0, dtype: TF_DataType.TF_INT32, name: "time"); + + + + Func? masking_fn; + Func? compute_masked_output = null; + if (mask != null) + { + if (go_backwards) + { + mask = tf.reverse(mask, axis: new[] { 0 }); + } + var mask_ta = tf.TensorArray(dtype: TF_DataType.TF_BOOL, size: time_steps_t); + mask_ta = mask_ta.unstack(mask); + + masking_fn = (time) => + { + return mask_ta.read(time); + }; + + compute_masked_output = (mask_t, flat_out, flat_mask) => + { + var tiled_mask_t = new Tensors(); + foreach (var o in flat_out) + { + tiled_mask_t.Add(_expand_mask(mask_t, o, fixed_dim: mask_t.rank)); + } + + Tensors res = new Tensors(); + foreach (var (m, o, fm) in zip(tiled_mask_t.ToList(), flat_out.ToList(), flat_mask.ToList())) + { + res.Add(tf.where(m, o, fm)); + } + return res; + }; + } + // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor)? + else if (input_length is Tensor) + { + if (go_backwards) + { + var max_len = tf.reduce_max(input_length, axis: 0); + var rev_input_length = tf.subtract(max_len - 1, input_length); + + masking_fn = (time) => + { + return tf.less(rev_input_length, time); + }; + } + else + { + masking_fn = (time) => + { + return tf.greater(input_length, time); + }; + } + + compute_masked_output = (mask_t, flat_out, flat_mask) => + { + var res = new List(); + foreach (var (o, zo) in zip(flat_out, flat_mask)) + { + res.Add(tf.where(mask_t, o, zo)); + } + return res; + }; + } + else + { + masking_fn = null; + } + + Func cond = (time) => (time < time_steps_t); + int parallel_iterations = 32; + if (masking_fn != null) + { + // Mask for the T output will be base on the output of T - 1. In the + // case T = 0, a zero filled tensor will be used. + var flat_zero_output = new Tensors(); + foreach (var o in Nest.Flatten(output_time_zero)) + { + flat_zero_output.Add(tf.zeros_like(o)); + } + + var prev_output = flat_zero_output; + var output_ta_t = output_ta; + Tensor _step(Tensor time) + { + /* + RNN step function. + Args: + time: Current timestep value. + output_ta_t: TensorArray. + prev_output: tuple of outputs from time - 1. + *states: List of states. + Returns: + Tuple(todo): `(time + 1, output_ta_t, output) + tuple(new_states)` + */ + + var flat_current_input = input_ta.Select(x => x.read(time)).ToList(); + // maybe set shape + // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + var current_input = Nest.PackSequenceAs(inputs, flat_current_input).ToTensors(); + var mask_t = masking_fn(time); + var (output, new_states_internal) = step_function(current_input, states.MergeWith(constants)); + // mask output + var flat_output = Nest.Flatten(output).ToList(); + + var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.ToList(); + + // TODO(Wanglongzhi2001),deal with compute_masked_output's third parameter's type + var flat_new_output = compute_masked_output(mask_t, flat_output, flat_mask_output); + + // mask states + var flat_state = states.ToList(); + var flat_new_state = new_states_internal.ToList(); + + foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + { + if (new_state is Tensor) + { + new_state.shape = state.shape; + } + } + + var flat_final_state = compute_masked_output(mask_t, flat_new_state, flat_state); + new_states_internal = Nest.PackSequenceAs(new_states, flat_final_state).ToTensors(); + + var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + // TODO(Wanglongzhi2001),deal with zip output_ta_t + foreach (var (ta, Out) in zip(output_ta_t, flat_new_output)) + { + output_ta_t.Add(ta.write(ta_index_to_write, Out)); + } + + new_states_internal = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); + + output_ta = output_ta_t; + new_states = new_states_internal; + return time + 1; + + } + var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: time, parallel_iterations: parallel_iterations); + } + else + { + var output_ta_t = output_ta; + new_states = states; + Tensor _step(Tensor time) + { + var flat_current_input = input_ta.Select(x => x.read(time)).ToList(); + // maybe set shape + // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type + var current_input = Nest.PackSequenceAs(inputs, flat_current_input).ToTensors(); + var (output, new_states_internal) = step_function(current_input, new_states.MergeWith(constants)); + var flat_state = new_states.Flatten().ToList(); + var flat_new_state = new_states_internal.Flatten().ToList(); + foreach (var (state, new_state) in zip(flat_state, flat_new_state)) + { + if (new_state is Tensor) + { + new_state.shape = state.shape; + } + } + var flat_output = Nest.Flatten(output); + var ta_index_to_write = return_all_outputs ? time : tf.constant(0); + output_ta_t = zip(output_ta_t, flat_output).Select(item => + { + var (ta, out_) = item; + return ta.write(ta_index_to_write, out_); + }).ToList(); + + new_states_internal = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); + output_ta = output_ta_t; + new_states = new_states_internal; + return time + 1; + } + var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: time, parallel_iterations: parallel_iterations); + } + //Tensors outputs = new Tensors(); + foreach (var o in output_ta) + { + outputs.Add(o.stack()); + } + foreach (var o in outputs) + { + last_output.Add(o[-1]); + } + outputs = Nest.PackSequenceAs(output_time_zero, outputs).ToTensors(); + last_output = Nest.PackSequenceAs(output_time_zero, last_output).ToTensors(); + } - //else // unroll == false - //{ - // var states = initial_states; - // // Create input tensor array, if the inputs is nested tensors, then it - // // will be flattened first, and tensor array will be created one per - // // flattened tensor. - // var input_ta = new List(); - // for (int i = 0; i < flatted_inptus.Count; i++) - // { - // input_ta.Add(tf.TensorArray(dtype: flatted_inptus[i].dtype, size: time_step_t)); - // } - - // // Get the time(0) input and compute the output for that, the output will - // // be used to determine the dtype of output tensor array. Don't read from - // // input_ta due to TensorArray clear_after_read default to True. - // var inps = new Tensors(); - // foreach (var inp in flatted_inptus) - // { - // inps.Add(inp[0]); - // } - // var input_time_zero = nest.pack_sequence_as(inputs, inps); - - // // output_time_zero is used to determine the cell output shape and its - // // dtype. the value is discarded. - // (output_time_zero, _) = step_function((Tensor)input_time_zero, new Tensors { initial_states, constants }); - - // var output_ta_size = return_all_outputs ? time_step_t : tf.constant(1); - // var output_ta = new List(); - // for (int i = 0; i < output_time_zero.ToList().Count; i++) - // { - // var Out = output_time_zero.ToList()[i]; - // output_ta.Add(tf.TensorArray(dtype: Out.dtype, size: output_ta_size, element_shape: Out.shape)); - // } - - // var time = tf.constant(0, dtype: TF_DataType.TF_INT32, name: "time"); - - - - // Func? masking_fn; - // Func? compute_masked_output = null; - // if (mask != null) - // { - // if (go_backwards) - // { - // mask = tf.reverse(mask, axis: new[] { 0 }); - // } - // var mask_ta = tf.TensorArray(dtype: TF_DataType.TF_BOOL, size: time_step_t); - // mask_ta = mask_ta.unstack(mask); - - // masking_fn = (time) => - // { - // return mask_ta.read(time); - // }; - - // compute_masked_output = (mask_t, flat_out, flat_mask) => - // { - // var tiled_mask_t = new Tensors(); - // foreach (var o in flat_out) - // { - // tiled_mask_t.Add(_expand_mask(mask_t, o, fixed_dim: mask_t.rank)); - // } - - // Tensors res = new Tensors(); - // foreach (var (m, o, fm) in Enumerable.Zip(tiled_mask_t, flat_out, flat_mask)) - // { - // res.Add(tf.where(m, o, fm)); - // } - // return res; - // }; - // } - // // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor)? - // else if (input_length is Tensor) - // { - // if (go_backwards) - // { - // var max_len = tf.reduce_max(input_length, axis: 0); - // var rev_input_length = tf.subtract(max_len - 1, input_length); - - // masking_fn = (time) => - // { - // return tf.less(rev_input_length, time); - // }; - // } - // else - // { - // masking_fn = (time) => - // { - // return tf.greater(input_length, time); - // }; - // } - - // compute_masked_output = (mask_t, flat_out, flat_mask) => - // { - // var res = new List(); - // foreach (var (o, zo) in zip(flat_out, flat_mask)) - // { - // res.Add(tf.where(mask_t, o, zo)); - // } - // return res; - // }; - // } - // else - // { - // masking_fn = null; - // } - - - // if (masking_fn != null) - // { - // // Mask for the T output will be base on the output of T - 1. In the - // // case T = 0, a zero filled tensor will be used. - // var flat_zero_output = new Tensors(); - // foreach (var o in nest.flatten(output_time_zero)) - // { - // flat_zero_output.Add(tf.zeros_like(o)); - // } - - - // (Tensor, List, Tensors, Tensors) _step(Tensor time, List output_ta_t, Tensors prev_output, Tensors states) - // { - // /* - // RNN step function. - // Args: - // time: Current timestep value. - // output_ta_t: TensorArray. - // prev_output: tuple of outputs from time - 1. - // *states: List of states. - // Returns: - // Tuple(todo): `(time + 1, output_ta_t, output) + tuple(new_states)` - // */ - - // var current_input = input_ta.Select(x => x.read(time)).ToList(); - // // maybe set shape - // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type - // current_input = (List)nest.pack_sequence_as(inputs, current_input); - // var mask_t = masking_fn(time); - // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); - // // mask output - // //var flat_output = nest.flatten(output); - // var flat_output = output.ToList(); - - // var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.ToList(); - - // // TODO(Wanglongzhi2001),deal with compute_masked_output's third parameter's type - // var flat_new_output = compute_masked_output(mask_t, flat_output, flat_mask_output); - - // // mask states - // var flat_state = states.ToList(); - // var flat_new_state = new_states.ToList(); - - // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) - // { - // if (new_state is Tensor) - // { - // new_state.set_shape(state.shape); - // } - // } - - // var flat_final_state = compute_masked_output(mask_t, flat_new_state, flat_state); - // new_states = (Tensors)nest.pack_sequence_as(new_states, flat_final_state); - - // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); - // var Output_ta_t = new List(); - // // TODO(Wanglongzhi2001),deal with zip output_ta_t - // foreach (var (ta, Out) in zip(output_ta_t, flat_new_output)) - // { - // Output_ta_t.Add(ta.write(ta_index_to_write, Out)); - // } - - - - // //new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); - - - // return (time + 1, Output_ta_t, flat_new_output, new_states); - - // } - // Func cond = (time) => (time < time_step_t); - - // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, flat_zero_output, states)); - // new_states = final_outputs.Item4; - // output_ta = final_outputs.Item2; - - // } - // else - // { - // (Tensor, List, Tensors) _step(Tensor time, List output_ta_t, Tensors states) - // { - // var current_input = input_ta.Select(x => x.read(time)).ToList(); - // // maybe set shape - // // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type - // current_input = (List)nest.pack_sequence_as(inputs, current_input); - // var (output, new_states) = step_function(current_input, new Tensors { states, constants }); - // var flat_state = states.ToList(); - // var flat_new_state = new_states.ToList(); - // foreach (var (state, new_state) in zip(flat_state, flat_new_state)) - // { - // if (new_state is Tensor) - // { - // new_state.set_shape(state.shape); - // } - // } - // var flat_output = output.ToList(); - // var ta_index_to_write = return_all_outputs ? time : tf.constant(0); - // var Output_ta_t = new List(); - // foreach (var (ta, out_) in zip(output_ta_t, flat_output)) - // { - // Output_ta_t.Add(ta.write(ta_index_to_write, out_)); - // } - - // new_states = (Tensors)nest.pack_sequence_as(initial_states, flat_new_state); - // return (time + 1, Output_ta_t, new_states); - // } - // Func cond = (time) => (time < time_step_t); - // var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: (time, output_ta, states)); - // new_states = final_outputs.Item3; - // output_ta = final_outputs.Item2; - - // } - // //Tensors outputs = new Tensors(); - // foreach (var o in output_ta) - // { - // outputs.Add(o.stack()); - // } - // foreach (var o in outputs) - // { - // last_output.Add(o[-1]); - // } - // outputs = (Tensors)nest.pack_sequence_as(output_time_zero, outputs); - // last_output = (Tensors)nest.pack_sequence_as(output_time_zero, last_output); - - //} Func set_shape; set_shape = (output_) => @@ -947,18 +950,38 @@ namespace Tensorflow.Keras shape[0] = 1; } shape[1] = (int)batch; - output_.set_shape(new Tensor(shape)); + output_.shape = shape; } return output_; }; - var Outputs = (Tensors)nest.map_structure(set_shape, outputs); + outputs = Nest.MapStructure(set_shape, outputs).ToTensors(); if (!time_major) { - Outputs = nest.map_structure(swap_batch_timestep, outputs); + outputs = Nest.MapStructure(swap_batch_timestep, outputs).ToTensors(); + } + return (last_output, outputs, new_states); + + } + + public Tensor reverse(Tensor input, int axis) + { + return reverse(input, new int[] { axis }); + } + + public Tensor reverse(Tensor input, int[] axes) + { + return tf.reverse(input, axes); + } + + public Tensor maybe_convert_to_ragged(bool is_ragged_output, Tensor output, int nested_row_lengths, bool go_backwards = false) + { + if (!is_ragged_output) + { + return output; } - return (last_output, Outputs, new_states); + throw new NotImplementedException("Not implemented currently, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index b014737f..ab4cef12 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -55,8 +55,8 @@ namespace Tensorflow.Keras.Layers.Rnn if (_states == null) { // CHECK(Rinne): check if this is correct. - var state = nest.map_structure(x => null, _cell.StateSize); - return new Tensors { state }; + var nested = _cell.StateSize.MapStructure(x => null); + _states = nested.AsNest().ToTensors(); } return _states; } @@ -230,7 +230,7 @@ namespace Tensorflow.Keras.Layers.Rnn Tensors? mask = rnn_optional_args?.Mask; //var (inputs_padded, row_length) = BackendImpl.convert_inputs_if_ragged(inputs); // 暂时先不接受ragged tensor - int? row_length = null; + int row_length = 0; // TODO(Rinne): support this param. bool is_ragged_input = false; _validate_args_if_ragged(is_ragged_input, mask); @@ -249,16 +249,16 @@ namespace Tensorflow.Keras.Layers.Rnn if (mask != null) { // Time step masks must be the same for each input. - mask = nest.flatten(mask)[0]; + mask = mask.Flatten().First(); } Shape input_shape; - if (nest.is_nested(inputs)) + if (!inputs.IsSingle()) { // In the case of nested input, use the first element for shape check // input_shape = nest.flatten(inputs)[0].shape; // TODO(Wanglongzhi2001) - input_shape = nest.flatten(inputs)[0].shape; + input_shape = inputs.Flatten().First().shape; } else { @@ -286,6 +286,7 @@ namespace Tensorflow.Keras.Layers.Rnn // cell_call_fn = (self.cell.__call__ if callable(self.cell) else self.cell.call) Func step; + bool is_tf_rnn_cell = _cell.IsTFRnnCell; if (constants is not null) { if (!_cell.SupportOptionalArgs) @@ -299,7 +300,8 @@ namespace Tensorflow.Keras.Layers.Rnn { constants = new Tensors(states.TakeLast(_num_constants)); states = new Tensors(states.SkipLast(_num_constants)); - var(output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); + states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; + var (output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); // TODO(Wanglongzhi2001),should cell_call_fn's return value be Tensors, Tensors? return (output, new_states.Single); }; @@ -308,13 +310,13 @@ namespace Tensorflow.Keras.Layers.Rnn { step = (inputs, states) => { - // states = (states[0] if len(states) == 1 and is_tf_rnn_cell else states) + states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = _cell.Apply(inputs, states); return (output, new_states.Single); }; } - var (last_output, outputs, states) = BackendImpl.rnn(step, + var (last_output, outputs, states) = keras.backend.rnn(step, inputs, initial_state, constants: constants, @@ -334,8 +336,8 @@ namespace Tensorflow.Keras.Layers.Rnn Tensors output = new Tensors(); if (_args.ReturnSequences) { - throw new NotImplementedException("this argument havn't been developed."); - + // TODO(Rinne): add go_backwards parameter and revise the `row_length` param + output = keras.backend.maybe_convert_to_ragged(is_ragged_input, outputs, row_length, false); } else { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs index fcb5d1eb..751312e5 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs @@ -14,8 +14,8 @@ namespace Tensorflow.Keras.Layers.Rnn public RnnCellBase(LayerArgs args) : base(args) { } public abstract GeneralizedTensorShape StateSize { get; } public abstract GeneralizedTensorShape OutputSize { get; } + public abstract bool IsTFRnnCell { get; } public abstract bool SupportOptionalArgs { get; } - public abstract (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null); public virtual Tensors GetInitialState(Tensors inputs, long batch_size, TF_DataType dtype) { return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index abb57d8a..f0b2ed4d 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -5,6 +5,7 @@ using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; +using Tensorflow.Common.Extensions; namespace Tensorflow.Keras.Layers.Rnn { @@ -26,6 +27,7 @@ namespace Tensorflow.Keras.Layers.Rnn public override GeneralizedTensorShape StateSize => _state_size; public override GeneralizedTensorShape OutputSize => _output_size; + public override bool IsTFRnnCell => true; public override bool SupportOptionalArgs => false; public SimpleRNNCell(SimpleRNNCellArgs args) : base(args) @@ -66,37 +68,22 @@ namespace Tensorflow.Keras.Layers.Rnn built = true; } - public override (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) + // TODO(Rinne): revise the trining param (with refactoring of the framework) + protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) { // TODO(Rinne): check if it will have multiple tensors when not nested. - Tensor prev_output = states[0]; + Tensors prev_output = Nest.IsNested(states) ? new Tensors(states[0]) : states; var dp_mask = get_dropout_maskcell_for_cell(inputs, training.Value); var rec_dp_mask = get_recurrent_dropout_maskcell_for_cell(prev_output, training.Value); Tensor h; - var ranks = inputs.rank; if (dp_mask != null) { - if (ranks > 2) - { - // 因为multiply函数会自动添加第一个维度，所以加上下标0 - h = tf.linalg.tensordot(math_ops.multiply(inputs, dp_mask)[0], _kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); - } - else - { - h = math_ops.matmul(math_ops.multiply(inputs, dp_mask)[0], _kernel.AsTensor()); - } + h = math_ops.matmul(math_ops.multiply(inputs.Single, dp_mask.Single), _kernel.AsTensor()); } else { - if (ranks > 2) - { - h = tf.linalg.tensordot(inputs, _kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); - } - else - { - h = math_ops.matmul(inputs, _kernel.AsTensor()); - } + h = math_ops.matmul(inputs, _kernel.AsTensor()); } if (_bias != null) @@ -106,26 +93,25 @@ namespace Tensorflow.Keras.Layers.Rnn if (rec_dp_mask != null) { - prev_output = math_ops.multiply(prev_output, rec_dp_mask)[0]; + prev_output = math_ops.multiply(prev_output, rec_dp_mask); } - ranks = prev_output.rank; - Tensor output; - if (ranks > 2) + Tensor output = h + math_ops.matmul(prev_output, _recurrent_kernel.AsTensor()); + + if (_args.Activation != null) { - output = h + tf.linalg.tensordot(prev_output[0], _recurrent_kernel.AsTensor(), new[,] { { ranks - 1 }, { 0 } }); + output = _args.Activation.Apply(output); } - else + if (Nest.IsNested(states)) { - output = h + math_ops.matmul(prev_output, _recurrent_kernel.AsTensor()); + return new Nest(new List> { + new Nest(new List> { new Nest(output) }), new Nest(output) }) + .ToTensors(); } - Console.WriteLine($"shape of output: {output.shape}"); - - if (_args.Activation != null) + else { - output = _args.Activation.Apply(output); + return new Tensors(output, output); } - return (output, new Tensors { output }); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 7923192f..0b92fd3c 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -170,6 +170,7 @@ namespace Tensorflow.Keras.Layers.Rnn } public GeneralizedTensorShape StateSize => throw new NotImplementedException(); public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); + public bool IsTFRnnCell => throw new NotImplementedException(); public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/src/TensorflowNET.Hub/KerasLayer.cs b/src/TensorflowNET.Hub/KerasLayer.cs index b9ca949b..20d9851b 100644 --- a/src/TensorflowNET.Hub/KerasLayer.cs +++ b/src/TensorflowNET.Hub/KerasLayer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Keras.Engine; using Tensorflow.Train; using Tensorflow.Training; @@ -89,7 +90,7 @@ namespace Tensorflow.Hub } } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optionalArgs = null) { _check_trainability(); From dcaa0f40d1f81b8e089f9ec77d85ca42e0933d80 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Wed, 7 Jun 2023 09:16:49 +0800 Subject: [PATCH 05/77] fix: some possible errors of RNN. --- src/TensorFlowNET.Core/Tensors/Tensors.cs | 41 +++++++++++++++++------ src/TensorFlowNET.Keras/BackendImpl.cs | 40 +++++++++------------- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensors/Tensors.cs b/src/TensorFlowNET.Core/Tensors/Tensors.cs index cba8f954..259b1eec 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensors.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensors.cs @@ -58,17 +58,12 @@ namespace Tensorflow public Tensor this[params string[] slices] => this.First()[slices]; - public Tensors(Tensor tensor) : base(tensor) - { - - } - private Tensors(Nest nested) : base(nested) { } - public Tensors(params Tensor[] tensors): base(tensors.Select(x => new Nest(x))) + public Tensors(params Tensor[] tensors): base(DealWithConstructorArrayInput(tensors)) { } @@ -83,6 +78,22 @@ namespace Tensorflow } + private static Nest DealWithConstructorArrayInput(Tensor[] tensors) + { + if (tensors.Length == 0) + { + return Nest.Empty; + } + else if(tensors.Length == 1) + { + return new Nest(tensors[0]); + } + else + { + return new Nest(tensors.Select(x => new Nest(x))); + } + } + public bool IsSingle() { return Length == 1; @@ -107,9 +118,14 @@ namespace Tensorflow ListValue = new() { new Nest(Value), new Nest(tensor) }; Value = null; } - else + else if(NestType == NestType.List) + { + ListValue!.Add(new Nest(tensor)); + } + else //Empty { - ListValue.Add(new Nest(tensor)); + NestType = NestType.Node; + Value = tensor; } } @@ -128,9 +144,14 @@ namespace Tensorflow ListValue.AddRange(tensors.Select(x => new Nest(x))); Value = null; } - else + else if(NestType == NestType.List) { - ListValue.AddRange(tensors.Select(x => new Nest(x))); + ListValue!.AddRange(tensors.Select(x => new Nest(x))); + } + else // empty + { + NestType = NestType.List; + ListValue = tensors.Select(x => new Nest(x)).ToList(); } } diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 30b73e82..14491066 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -651,13 +651,13 @@ namespace Tensorflow.Keras states = Nest.PackSequenceAs(states, flat_final_states).ToTensors(); if (return_all_outputs) { - successive_outputs.Add(output); - successive_states.Add(states); + successive_outputs = successive_outputs.MergeWith(output); + successive_outputs = successive_states.MergeWith(states); } else { - successive_outputs = new Tensors { output }; - successive_states = new Tensors { states }; + successive_outputs = new Tensors(output); + successive_states = new Tensors(states); } } @@ -722,16 +722,11 @@ namespace Tensorflow.Keras // Get the time(0) input and compute the output for that, the output will // be used to determine the dtype of output tensor array. Don't read from // input_ta due to TensorArray clear_after_read default to True. - var inps = new Tensors(); - foreach (var inp in flatted_inptus) - { - inps.Add(inp[0]); - } - var input_time_zero = Nest.PackSequenceAs(inputs, inps).ToTensors(); + var input_time_zero = Nest.PackSequenceAs(inputs, flatted_inptus.Select(x => x[0]).ToArray()).ToTensors(); // output_time_zero is used to determine the cell output shape and its // dtype. the value is discarded. - (output_time_zero, _) = step_function((Tensor)input_time_zero, + (output_time_zero, _) = step_function(input_time_zero, constants is null ? initial_states : initial_states.MergeWith(constants)); int output_ta_size = return_all_outputs ? time_steps_t : 1; @@ -816,6 +811,7 @@ namespace Tensorflow.Keras Func cond = (time) => (time < time_steps_t); int parallel_iterations = 32; + new_states = states; if (masking_fn != null) { // Mask for the T output will be base on the output of T - 1. In the @@ -846,7 +842,7 @@ namespace Tensorflow.Keras // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type var current_input = Nest.PackSequenceAs(inputs, flat_current_input).ToTensors(); var mask_t = masking_fn(time); - var (output, new_states_internal) = step_function(current_input, states.MergeWith(constants)); + var (output, new_states_internal) = step_function(current_input, new_states.MergeWith(constants)); // mask output var flat_output = Nest.Flatten(output).ToList(); @@ -871,11 +867,12 @@ namespace Tensorflow.Keras new_states_internal = Nest.PackSequenceAs(new_states, flat_final_state).ToTensors(); var ta_index_to_write = return_all_outputs ? time : tf.constant(0); - // TODO(Wanglongzhi2001),deal with zip output_ta_t - foreach (var (ta, Out) in zip(output_ta_t, flat_new_output)) + output_ta_t = zip(output_ta_t, flat_new_output).Select(item => { - output_ta_t.Add(ta.write(ta_index_to_write, Out)); - } + var (ta, out_) = item; + return ta.write(ta_index_to_write, out_); + }).ToList(); + new_states_internal = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); @@ -921,15 +918,8 @@ namespace Tensorflow.Keras } var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: time, parallel_iterations: parallel_iterations); } - //Tensors outputs = new Tensors(); - foreach (var o in output_ta) - { - outputs.Add(o.stack()); - } - foreach (var o in outputs) - { - last_output.Add(o[-1]); - } + outputs = outputs.MergeWith(output_ta.Select(o => o.stack()).ToTensors()); + last_output = last_output.MergeWith(outputs.Select(o => o[-1]).ToTensors()); outputs = Nest.PackSequenceAs(output_time_zero, outputs).ToTensors(); last_output = Nest.PackSequenceAs(output_time_zero, last_output).ToTensors(); From db8e43b241cbc86a707bab7f0da5d4a0861820ec Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Mon, 12 Jun 2023 17:59:07 +0800 Subject: [PATCH 06/77] Add feature(not completed):add SimpleRNNCell, StackedRNNCell, RNN and test --- .../Common/Types/GeneralizedTensorShape.cs | 14 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 3 + .../ArgsDefinition/Rnn/StackedRNNCellsArgs.cs | 3 +- .../Keras/Layers/ILayersApi.cs | 34 ++++ .../Operations/_EagerTensorArray.cs | 14 +- .../Operations/_GraphTensorArray.cs | 5 +- src/TensorFlowNET.Keras/BackendImpl.cs | 27 +-- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 77 +++++++++ .../Layers/Rnn/DropoutRNNCellMixin.cs | 15 ++ src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 76 ++++++--- .../Layers/Rnn/SimpleRNNCell.cs | 10 +- .../Layers/Rnn/StackedRNNCells.cs | 159 +++++++++++------- .../Callbacks/EarlystoppingTest.cs | 25 ++- .../Layers/Rnn.Test.cs | 102 ++++++++++- 14 files changed, 445 insertions(+), 119 deletions(-) diff --git a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs index e05d3deb..c61d04b2 100644 --- a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs +++ b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs @@ -12,9 +12,14 @@ namespace Tensorflow.Common.Types /// create a single-dim generalized Tensor shape. /// /// - public GeneralizedTensorShape(int dim) + public GeneralizedTensorShape(int dim, int size = 1) { - Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; + var elem = new TensorShapeConfig() { Items = new long?[] { dim } }; + Shapes = Enumerable.Repeat(elem, size).ToArray(); + //Shapes = new TensorShapeConfig[size]; + //Shapes.Initialize(new TensorShapeConfig() { Items = new long?[] { dim } }); + //Array.Initialize(Shapes, new TensorShapeConfig() { Items = new long?[] { dim } }); + ////Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; } public GeneralizedTensorShape(Shape shape) @@ -113,6 +118,11 @@ namespace Tensorflow.Common.Types return new Nest(Shapes.Select(s => DealWithSingleShape(s))); } } + + + + public static implicit operator GeneralizedTensorShape(int dims) + => new GeneralizedTensorShape(dims); public IEnumerator GetEnumerator() { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index ed5a1d6d..116ff7a2 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -10,6 +10,9 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn [JsonProperty("cell")] // TODO: the cell should be serialized with `serialize_keras_object`. public IRnnCell Cell { get; set; } = null; + [JsonProperty("cells")] + public IList Cells { get; set; } = null; + [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; [JsonProperty("return_state")] diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index fdfadab8..ea6f830b 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -1,10 +1,11 @@ using System.Collections.Generic; +using Tensorflow.Keras.Layers.Rnn; namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class StackedRNNCellsArgs : LayerArgs { - public IList Cells { get; set; } + public IList Cells { get; set; } public Dictionary Kwargs { get; set; } = null; } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 6a29f9e5..3b223816 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,5 +1,6 @@ using System; using Tensorflow.Framework.Models; +using Tensorflow.Keras.Layers.Rnn; using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; @@ -192,6 +193,19 @@ namespace Tensorflow.Keras.Layers float offset = 0, Shape input_shape = null); + public IRnnCell SimpleRNNCell( + int units, + string activation = "tanh", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f); + + public IRnnCell StackedRNNCells( + IEnumerable cells); + public ILayer SimpleRNN(int units, string activation = "tanh", string kernel_initializer = "glorot_uniform", @@ -200,6 +214,26 @@ namespace Tensorflow.Keras.Layers bool return_sequences = false, bool return_state = false); + public ILayer RNN( + IRnnCell cell, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false + ); + + public ILayer RNN( + IEnumerable cell, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false + ); + public ILayer Subtract(); } } diff --git a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs index ed65a08d..08e73fe6 100644 --- a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs @@ -109,7 +109,19 @@ namespace Tensorflow.Operations return ta; });*/ - throw new NotImplementedException(""); + //if (indices is EagerTensor) + //{ + // indices = indices as EagerTensor; + // indices = indices.numpy(); + //} + + //foreach (var (index, val) in zip(indices.ToArray(), array_ops.unstack(value))) + //{ + // this.write(index, val); + //} + //return base; + //throw new NotImplementedException(""); + return this; } public void _merge_element_shape(Shape shape) diff --git a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs index 16870e9f..dde2624a 100644 --- a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Eager; using static Tensorflow.Binding; namespace Tensorflow.Operations @@ -146,7 +147,9 @@ namespace Tensorflow.Operations return ta; });*/ - throw new NotImplementedException(""); + + //throw new NotImplementedException(""); + return this; } public void _merge_element_shape(Shape shape) diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 14491066..1336e9af 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -510,7 +510,7 @@ namespace Tensorflow.Keras } } - + // tf.where needs its condition tensor to be the same shape as its two // result tensors, but in our case the condition (mask) tensor is // (nsamples, 1), and inputs are (nsamples, ndimensions) or even more. @@ -535,7 +535,7 @@ namespace Tensorflow.Keras { mask_t = tf.expand_dims(mask_t, -1); } - var multiples = Enumerable.Repeat(1, fixed_dim).ToArray().concat(input_t.shape.as_int_list().ToList().GetRange(fixed_dim, input_t.rank)); + var multiples = Enumerable.Repeat(1, fixed_dim).ToArray().concat(input_t.shape.as_int_list().Skip(fixed_dim).ToArray()); return tf.tile(mask_t, multiples); } @@ -570,9 +570,6 @@ namespace Tensorflow.Keras // individually. The result of this will be a tuple of lists, each of // the item in tuple is list of the tensor with shape (batch, feature) - - - Tensors _process_single_input_t(Tensor input_t) { var unstaked_input_t = array_ops.unstack(input_t); // unstack for time_step dim @@ -609,7 +606,7 @@ namespace Tensorflow.Keras var mask_list = tf.unstack(mask); if (go_backwards) { - mask_list.Reverse(); + mask_list.Reverse().ToArray(); } for (int i = 0; i < time_steps; i++) @@ -629,9 +626,10 @@ namespace Tensorflow.Keras } else { - prev_output = successive_outputs[successive_outputs.Length - 1]; + prev_output = successive_outputs.Last(); } + // output could be a tensor output = tf.where(tiled_mask_t, output, prev_output); var flat_states = Nest.Flatten(states).ToList(); @@ -661,13 +659,13 @@ namespace Tensorflow.Keras } } - last_output = successive_outputs[successive_outputs.Length - 1]; - new_states = successive_states[successive_states.Length - 1]; + last_output = successive_outputs.Last(); + new_states = successive_states.Last(); outputs = tf.stack(successive_outputs); if (zero_output_for_mask) { - last_output = tf.where(_expand_mask(mask_list[mask_list.Length - 1], last_output), last_output, tf.zeros_like(last_output)); + last_output = tf.where(_expand_mask(mask_list.Last(), last_output), last_output, tf.zeros_like(last_output)); outputs = tf.where(_expand_mask(mask, outputs, fixed_dim: 2), outputs, tf.zeros_like(outputs)); } else // mask is null @@ -689,8 +687,8 @@ namespace Tensorflow.Keras successive_states = new Tensors { newStates }; } } - last_output = successive_outputs[successive_outputs.Length - 1]; - new_states = successive_states[successive_states.Length - 1]; + last_output = successive_outputs.Last(); + new_states = successive_states.Last(); outputs = tf.stack(successive_outputs); } } @@ -701,6 +699,8 @@ namespace Tensorflow.Keras // Create input tensor array, if the inputs is nested tensors, then it // will be flattened first, and tensor array will be created one per // flattened tensor. + + var input_ta = new List(); for (int i = 0; i < flatted_inptus.Count; i++) { @@ -719,6 +719,7 @@ namespace Tensorflow.Keras } } + // Get the time(0) input and compute the output for that, the output will // be used to determine the dtype of output tensor array. Don't read from // input_ta due to TensorArray clear_after_read default to True. @@ -773,7 +774,7 @@ namespace Tensorflow.Keras return res; }; } - // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor)? + // TODO(Wanglongzhi2001), what the input_length's type should be(an integer or a single tensor), it could be an integer or tensor else if (input_length is Tensor) { if (go_backwards) diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 3b095bc2..dd25122d 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -685,6 +685,34 @@ namespace Tensorflow.Keras.Layers Alpha = alpha }); + + public IRnnCell SimpleRNNCell( + int units, + string activation = "tanh", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f) + => new SimpleRNNCell(new SimpleRNNCellArgs + { + Units = units, + Activation = keras.activations.GetActivationFromName(activation), + UseBias = use_bias, + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + Dropout = dropout, + RecurrentDropout = recurrent_dropout + }); + + public IRnnCell StackedRNNCells( + IEnumerable cells) + => new StackedRNNCells(new StackedRNNCellsArgs + { + Cells = cells.ToList() + }); + /// /// /// @@ -709,6 +737,55 @@ namespace Tensorflow.Keras.Layers ReturnState = return_state }); + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public ILayer RNN( + IRnnCell cell, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false) + => new RNN(new RNNArgs + { + Cell = cell, + ReturnSequences = return_sequences, + ReturnState = return_state, + GoBackwards = go_backwards, + Stateful = stateful, + Unroll = unroll, + TimeMajor = time_major + }); + + public ILayer RNN( + IEnumerable cell, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false) + => new RNN(new RNNArgs + { + Cells = cell.ToList(), + ReturnSequences = return_sequences, + ReturnState = return_state, + GoBackwards = go_backwards, + Stateful = stateful, + Unroll = unroll, + TimeMajor = time_major + }); + /// /// Long Short-Term Memory layer - Hochreiter 1997. /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 21396853..78d3dac9 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -17,6 +17,21 @@ namespace Tensorflow.Keras.Layers.Rnn } + protected void _create_non_trackable_mask_cache() + { + + } + + public void reset_dropout_mask() + { + + } + + public void reset_recurrent_dropout_mask() + { + + } + public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) { if (dropout == 0f) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index ab4cef12..0ebd7362 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -38,7 +38,17 @@ namespace Tensorflow.Keras.Layers.Rnn SupportsMasking = true; // if is StackedRnncell - _cell = args.Cell; + if (args.Cells != null) + { + _cell = new StackedRNNCells(new StackedRNNCellsArgs + { + Cells = args.Cells + }); + } + else + { + _cell = args.Cell; + } // get input_shape _args = PreConstruct(args); @@ -122,6 +132,8 @@ namespace Tensorflow.Keras.Layers.Rnn var state_shape = new int[] { (int)batch }.concat(flat_state.as_int_list()); return new Shape(state_shape); }; + + var state_shape = _get_state_shape(state_size); return new List { output_shape, state_shape }; @@ -240,7 +252,7 @@ namespace Tensorflow.Keras.Layers.Rnn if (_cell is StackedRNNCells) { var stack_cell = _cell as StackedRNNCells; - foreach (var cell in stack_cell.Cells) + foreach (IRnnCell cell in stack_cell.Cells) { _maybe_reset_cell_dropout_mask(cell); } @@ -253,7 +265,7 @@ namespace Tensorflow.Keras.Layers.Rnn } Shape input_shape; - if (!inputs.IsSingle()) + if (!inputs.IsNested()) { // In the case of nested input, use the first element for shape check // input_shape = nest.flatten(inputs)[0].shape; @@ -267,7 +279,7 @@ namespace Tensorflow.Keras.Layers.Rnn var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; - if (_args.Unroll && timesteps != null) + if (_args.Unroll && timesteps == null) { throw new ValueError( "Cannot unroll a RNN if the " + @@ -302,7 +314,6 @@ namespace Tensorflow.Keras.Layers.Rnn states = new Tensors(states.SkipLast(_num_constants)); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); - // TODO(Wanglongzhi2001),should cell_call_fn's return value be Tensors, Tensors? return (output, new_states.Single); }; } @@ -310,13 +321,14 @@ namespace Tensorflow.Keras.Layers.Rnn { step = (inputs, states) => { - states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; + states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states.First()) : states; var (output, new_states) = _cell.Apply(inputs, states); - return (output, new_states.Single); + return (output, new_states); }; } - - var (last_output, outputs, states) = keras.backend.rnn(step, + + var (last_output, outputs, states) = keras.backend.rnn( + step, inputs, initial_state, constants: constants, @@ -394,6 +406,7 @@ namespace Tensorflow.Keras.Layers.Rnn initial_state = null; inputs = inputs[0]; } + if (_args.Stateful) { @@ -402,7 +415,7 @@ namespace Tensorflow.Keras.Layers.Rnn var tmp = new Tensor[] { }; foreach (var s in nest.flatten(States)) { - tmp.add(tf.math.count_nonzero((Tensor)s)); + tmp.add(tf.math.count_nonzero(s.Single())); } var non_zero_count = tf.add_n(tmp); //initial_state = tf.cond(non_zero_count > 0, () => States, () => initial_state); @@ -415,6 +428,15 @@ namespace Tensorflow.Keras.Layers.Rnn { initial_state = States; } + // TODO(Wanglongzhi2001), +// initial_state = tf.nest.map_structure( +//# When the layer has a inferred dtype, use the dtype from the +//# cell. +// lambda v: tf.cast( +// v, self.compute_dtype or self.cell.compute_dtype +// ), +// initial_state, +// ) } else if (initial_state is null) @@ -424,10 +446,9 @@ namespace Tensorflow.Keras.Layers.Rnn if (initial_state.Length != States.Length) { - throw new ValueError( - $"Layer {this} expects {States.Length} state(s), " + - $"but it received {initial_state.Length} " + - $"initial state(s). Input received: {inputs}"); + throw new ValueError($"Layer {this} expects {States.Length} state(s), " + + $"but it received {initial_state.Length} " + + $"initial state(s). Input received: {inputs}"); } return (inputs, initial_state, constants); @@ -458,11 +479,11 @@ namespace Tensorflow.Keras.Layers.Rnn void _maybe_reset_cell_dropout_mask(ILayer cell) { - //if (cell is DropoutRNNCellMixin) - //{ - // cell.reset_dropout_mask(); - // cell.reset_recurrent_dropout_mask(); - //} + if (cell is DropoutRNNCellMixin CellDRCMixin) + { + CellDRCMixin.reset_dropout_mask(); + CellDRCMixin.reset_recurrent_dropout_mask(); + } } private static RNNArgs PreConstruct(RNNArgs args) @@ -537,15 +558,24 @@ namespace Tensorflow.Keras.Layers.Rnn protected Tensors get_initial_state(Tensors inputs) { + var get_initial_state_fn = _cell.GetType().GetMethod("get_initial_state"); + var input = inputs[0]; - var input_shape = input.shape; + var input_shape = inputs.shape; var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; var dtype = input.dtype; - Tensors init_state; - if (_cell is RnnCellBase rnn_base_cell) + + Tensors init_state = new Tensors(); + + if(get_initial_state_fn != null) { - init_state = rnn_base_cell.GetInitialState(null, batch_size, dtype); + init_state = (Tensors)get_initial_state_fn.Invoke(_cell, new object[] { inputs, batch_size, dtype }); + } + //if (_cell is RnnCellBase rnn_base_cell) + //{ + // init_state = rnn_base_cell.GetInitialState(null, batch_size, dtype); + //} else { init_state = RnnUtils.generate_zero_filled_state(batch_size, _cell.StateSize, dtype); diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index f0b2ed4d..39610ff5 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -6,6 +6,7 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Layers.Rnn { @@ -77,8 +78,10 @@ namespace Tensorflow.Keras.Layers.Rnn var rec_dp_mask = get_recurrent_dropout_maskcell_for_cell(prev_output, training.Value); Tensor h; + var ranks = inputs.rank; if (dp_mask != null) { + h = math_ops.matmul(math_ops.multiply(inputs.Single, dp_mask.Single), _kernel.AsTensor()); } else @@ -95,7 +98,7 @@ namespace Tensorflow.Keras.Layers.Rnn { prev_output = math_ops.multiply(prev_output, rec_dp_mask); } - + var tmp = _recurrent_kernel.AsTensor(); Tensor output = h + math_ops.matmul(prev_output, _recurrent_kernel.AsTensor()); if (_args.Activation != null) @@ -113,5 +116,10 @@ namespace Tensorflow.Keras.Layers.Rnn return new Tensors(output, output); } } + + public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) + { + return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size.Value, dtype.Value); + } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 0b92fd3c..56634853 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -1,17 +1,20 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Linq; +using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Layers.Rnn { public class StackedRNNCells : Layer, IRnnCell { - public IList Cells { get; set; } + public IList Cells { get; set; } public bool reverse_state_order; public StackedRNNCells(StackedRNNCellsArgs args) : base(args) @@ -20,8 +23,19 @@ namespace Tensorflow.Keras.Layers.Rnn { args.Kwargs = new Dictionary(); } - + foreach (var cell in args.Cells) + { + //Type type = cell.GetType(); + //var CallMethodInfo = type.GetMethod("Call"); + //if (CallMethodInfo == null) + //{ + // throw new ValueError( + // "All cells must have a `Call` method. " + + // $"Received cell without a `Call` method: {cell}"); + //} + } Cells = args.Cells; + reverse_state_order = (bool)args.Kwargs.Get("reverse_state_order", false); if (reverse_state_order) @@ -33,91 +47,112 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public object state_size + public GeneralizedTensorShape StateSize { - get => throw new NotImplementedException(); - //@property - //def state_size(self) : - // return tuple(c.state_size for c in - // (self.cells[::- 1] if self.reverse_state_order else self.cells)) + get + { + GeneralizedTensorShape state_size = new GeneralizedTensorShape(1, Cells.Count); + if (reverse_state_order && Cells.Count > 0) + { + var idxAndCell = Cells.Reverse().Select((cell, idx) => (idx, cell)); + foreach (var cell in idxAndCell) + { + state_size.Shapes[cell.idx] = cell.cell.StateSize.Shapes.First(); + } + } + else + { + //foreach (var cell in Cells) + //{ + // state_size.Shapes.add(cell.StateSize.Shapes.First()); + + //} + var idxAndCell = Cells.Select((cell, idx) => (idx, cell)); + foreach (var cell in idxAndCell) + { + state_size.Shapes[cell.idx] = cell.cell.StateSize.Shapes.First(); + } + } + return state_size; + } } public object output_size { get { - var lastCell = Cells[Cells.Count - 1]; - - if (lastCell.output_size != -1) + var lastCell = Cells.LastOrDefault(); + if (lastCell.OutputSize.ToSingleShape() != -1) { - return lastCell.output_size; + return lastCell.OutputSize; } else if (RNN.is_multiple_state(lastCell.StateSize)) { - // return ((dynamic)Cells[-1].state_size)[0]; - throw new NotImplementedException(""); + return lastCell.StateSize.First(); + //throw new NotImplementedException(""); } else { - return Cells[-1].state_size; + return lastCell.StateSize; } } } - public object get_initial_state() + public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) { - throw new NotImplementedException(); - // def get_initial_state(self, inputs= None, batch_size= None, dtype= None) : - // initial_states = [] - // for cell in self.cells[::- 1] if self.reverse_state_order else self.cells: - // get_initial_state_fn = getattr(cell, 'get_initial_state', None) - // if get_initial_state_fn: - // initial_states.append(get_initial_state_fn( - // inputs=inputs, batch_size=batch_size, dtype=dtype)) - // else: - // initial_states.append(_generate_zero_filled_state_for_cell( - // cell, inputs, batch_size, dtype)) - - // return tuple(initial_states) + var cells = reverse_state_order ? Cells.Reverse() : Cells; + Tensors initial_states = new Tensors(); + foreach (var cell in cells) + { + var get_initial_state_fn = cell.GetType().GetMethod("get_initial_state"); + if (get_initial_state_fn != null) + { + var result = (Tensors)get_initial_state_fn.Invoke(cell, new object[] { inputs, batch_size, dtype }); + initial_states.Add(result); + } + else + { + initial_states.Add(RnnUtils.generate_zero_filled_state_for_cell(cell, inputs, batch_size.Value, dtype.Value)); + } + } + return initial_states; } - public object call() + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { - throw new NotImplementedException(); - // def call(self, inputs, states, constants= None, training= None, ** kwargs): - // # Recover per-cell states. - // state_size = (self.state_size[::- 1] - // if self.reverse_state_order else self.state_size) - // nested_states = nest.pack_sequence_as(state_size, nest.flatten(states)) - - // # Call the cells in order and store the returned states. - // new_nested_states = [] - // for cell, states in zip(self.cells, nested_states) : - // states = states if nest.is_nested(states) else [states] - //# TF cell does not wrap the state into list when there is only one state. - // is_tf_rnn_cell = getattr(cell, '_is_tf_rnn_cell', None) is not None - // states = states[0] if len(states) == 1 and is_tf_rnn_cell else states - // if generic_utils.has_arg(cell.call, 'training'): - // kwargs['training'] = training - // else: - // kwargs.pop('training', None) - // # Use the __call__ function for callable objects, eg layers, so that it - // # will have the proper name scopes for the ops, etc. - // cell_call_fn = cell.__call__ if callable(cell) else cell.call - // if generic_utils.has_arg(cell.call, 'constants'): - // inputs, states = cell_call_fn(inputs, states, - // constants= constants, ** kwargs) - // else: - // inputs, states = cell_call_fn(inputs, states, ** kwargs) - // new_nested_states.append(states) + // Recover per-cell states. + var state_size = reverse_state_order ? StateSize.Reverse() : StateSize; + var nested_states = reverse_state_order ? state.Flatten().Reverse() : state.Flatten(); - // return inputs, nest.pack_sequence_as(state_size, - // nest.flatten(new_nested_states)) + + var new_nest_states = new Tensors(); + // Call the cells in order and store the returned states. + foreach (var (cell, states) in zip(Cells, nested_states)) + { + // states = states if tf.nest.is_nested(states) else [states] + var type = cell.GetType(); + bool IsTFRnnCell = type.GetProperty("IsTFRnnCell") != null; + state = len(state) == 1 && IsTFRnnCell ? state.FirstOrDefault() : state; + + RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; + Tensors? constants = rnn_optional_args?.Constants; + + Tensors new_states; + (inputs, new_states) = cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); + + new_nest_states.Add(new_states); + } + new_nest_states = reverse_state_order ? new_nest_states.Reverse().ToArray() : new_nest_states.ToArray(); + return new Nest(new List> { + new Nest(new List> { new Nest(inputs.Single()) }), new Nest(new_nest_states) }) + .ToTensors(); } + + public void build() { - throw new NotImplementedException(); + built = true; // @tf_utils.shape_type_conversion // def build(self, input_shape) : // if isinstance(input_shape, list) : @@ -168,9 +203,9 @@ namespace Tensorflow.Keras.Layers.Rnn { throw new NotImplementedException(); } - public GeneralizedTensorShape StateSize => throw new NotImplementedException(); + public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); - public bool IsTFRnnCell => throw new NotImplementedException(); + public bool IsTFRnnCell => true; public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Callbacks/EarlystoppingTest.cs b/test/TensorFlowNET.Keras.UnitTest/Callbacks/EarlystoppingTest.cs index ac5ba15e..29648790 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Callbacks/EarlystoppingTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Callbacks/EarlystoppingTest.cs @@ -2,6 +2,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Collections.Generic; using Tensorflow.Keras.Callbacks; using Tensorflow.Keras.Engine; +using Tensorflow.NumPy; using static Tensorflow.KerasApi; @@ -18,7 +19,7 @@ namespace Tensorflow.Keras.UnitTest.Callbacks var layers = keras.layers; var model = keras.Sequential(new List { - layers.Rescaling(1.0f / 255, input_shape: (32, 32, 3)), + layers.Rescaling(1.0f / 255, input_shape: (28, 28, 1)), layers.Conv2D(32, 3, padding: "same", activation: keras.activations.Relu), layers.MaxPooling2D(), layers.Flatten(), @@ -36,8 +37,20 @@ namespace Tensorflow.Keras.UnitTest.Callbacks var num_epochs = 3; var batch_size = 8; - var ((x_train, y_train), (x_test, y_test)) = keras.datasets.cifar10.load_data(); - x_train = x_train / 255.0f; + var data_loader = new MnistModelLoader(); + + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 59900, + }).Result; + + NDArray x1 = np.reshape(dataset.Train.Data, (dataset.Train.Data.shape[0], 28, 28, 1)); + NDArray x2 = x1; + + var x = new NDArray[] { x1, x2 }; + // define a CallbackParams first, the parameters you pass al least contain Model and Epochs. CallbackParams callback_parameters = new CallbackParams { @@ -47,10 +60,8 @@ namespace Tensorflow.Keras.UnitTest.Callbacks // define your earlystop ICallback earlystop = new EarlyStopping(callback_parameters, "accuracy"); // define a callbcaklist, then add the earlystopping to it. - var callbacks = new List(); - callbacks.add(earlystop); - - model.fit(x_train[new Slice(0, 2000)], y_train[new Slice(0, 2000)], batch_size, num_epochs, callbacks: callbacks); + var callbacks = new List{ earlystop}; + model.fit(x, dataset.Train.Labels, batch_size, num_epochs, callbacks: callbacks); } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 55663d41..28a16ad4 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -4,25 +4,111 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Saving; using Tensorflow.NumPy; +using Tensorflow.Train; using static Tensorflow.Binding; +using static Tensorflow.KerasApi; namespace Tensorflow.Keras.UnitTest.Layers { [TestClass] public class Rnn { + [TestMethod] + public void SimpleRNNCell() + { + //var cell = tf.keras.layers.SimpleRNNCell(64, dropout: 0.5f, recurrent_dropout: 0.5f); + //var h0 = new Tensors { tf.zeros(new Shape(4, 64)) }; + //var x = tf.random.normal((4, 100)); + //var (y, h1) = cell.Apply(inputs: x, states: h0); + //var h2 = h1; + //Assert.AreEqual((4, 64), y.shape); + //Assert.AreEqual((4, 64), h2[0].shape); + + //var model = keras.Sequential(new List + //{ + // keras.layers.InputLayer(input_shape: (4,100)), + // keras.layers.SimpleRNNCell(64) + //}); + //model.summary(); + + var cell = tf.keras.layers.SimpleRNNCell(64, dropout: 0.5f, recurrent_dropout: 0.5f); + var h0 = new Tensors { tf.zeros(new Shape(4, 64)) }; + var x = tf.random.normal((4, 100)); + var (y, h1) = cell.Apply(inputs: x, states: h0); + var h2 = h1; + Assert.AreEqual((4, 64), y.shape); + Assert.AreEqual((4, 64), h2[0].shape); + } + + [TestMethod] + public void StackedRNNCell() + { + var inputs = tf.ones((32, 10)); + var states = new Tensors { tf.zeros((32, 4)), tf.zeros((32, 5)) }; + var cells = new IRnnCell[] { tf.keras.layers.SimpleRNNCell(4), tf.keras.layers.SimpleRNNCell(5) }; + var stackedRNNCell = tf.keras.layers.StackedRNNCells(cells); + var (output, state) = stackedRNNCell.Apply(inputs, states); + Console.WriteLine(output); + Console.WriteLine(state.shape); + Assert.AreEqual((32, 5), output.shape); + Assert.AreEqual((32, 4), state[0].shape); + } + [TestMethod] public void SimpleRNN() { - var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); - /*var simple_rnn = keras.layers.SimpleRNN(4); - var output = simple_rnn.Apply(inputs); - Assert.AreEqual((32, 4), output.shape);*/ - var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); - var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); - Console.WriteLine(whole_sequence_output); - Console.WriteLine(final_state); + //var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); + ///*var simple_rnn = keras.layers.SimpleRNN(4); + //var output = simple_rnn.Apply(inputs); + //Assert.AreEqual((32, 4), output.shape);*/ + + //var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); + //var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); + //Assert.AreEqual((6, 10, 4), whole_sequence_output.shape); + //Assert.AreEqual((6, 4), final_state.shape); + + var inputs = keras.Input(shape: (10, 8)); + var x = keras.layers.SimpleRNN(4).Apply(inputs); + var output = keras.layers.Dense(10).Apply(x); + var model = keras.Model(inputs, output); + model.summary(); + } + [TestMethod] + public void RNNForSimpleRNNCell() + { + var inputs = tf.random.normal((32, 10, 8)); + var cell = tf.keras.layers.SimpleRNNCell(10, dropout: 0.5f, recurrent_dropout: 0.5f); + var rnn = tf.keras.layers.RNN(cell: cell); + var output = rnn.Apply(inputs); + Assert.AreEqual((32, 10), output.shape); + } + [TestMethod] + public void RNNForStackedRNNCell() + { + var inputs = tf.random.normal((32, 10, 8)); + var cells = new IRnnCell[] { tf.keras.layers.SimpleRNNCell(4), tf.keras.layers.SimpleRNNCell(5) }; + var stackedRNNCell = tf.keras.layers.StackedRNNCells(cells); + var rnn = tf.keras.layers.RNN(cell: stackedRNNCell); + var output = rnn.Apply(inputs); + Assert.AreEqual((32, 5), output.shape); + } + + [TestMethod] + public void WlzTest() + { + long[] b = { 1, 2, 3 }; + + Shape a = new Shape(Unknown).concatenate(b); + Console.WriteLine(a); + + } + + } } From f1fbcf20166fa1902e399998aaf1c738493f9785 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 16 Jun 2023 14:30:54 +0800 Subject: [PATCH 07/77] feat: support model building with RNN. --- src/TensorFlowNET.Core/APIs/c_api.cs | 14 + .../APIs/tf.control_flow.cs | 10 +- .../Common/Extensions/LinqExtensions.cs | 7 +- .../Common/Types/FakeTensorByTensorArray.cs | 20 + .../Common/Types/GeneralizedTensorShape.cs | 140 +- .../Types/{INest.cs => INestStructure.cs} | 13 + .../Common/Types/Nest.Static.cs | 2 +- src/TensorFlowNET.Core/Common/Types/Nest.cs | 117 +- .../Common/Types/NestDictionary.cs | 4 + .../Common/Types/NestList.cs | 17 +- .../Common/Types/NestNode.cs | 4 + src/TensorFlowNET.Core/Data/DatasetV2.cs | 4 +- .../Eager/EagerRunner.TFE_FastPathExecute.cs | 2 + .../Framework/Models/TensorSpec.cs | 13 + .../Framework/auto_control_deps_utils.cs | 89 ++ .../Framework/function_def_lib.cs | 4 +- .../Functions/ConcreteFunction.cs | 13 + src/TensorFlowNET.Core/Graphs/FuncGraph.cs | 4 +- src/TensorFlowNET.Core/Graphs/Graph.cs | 2 +- .../Keras/Layers/Rnn/IRnnCell.cs | 12 +- .../Operations/NnOps/RNNCell.cs | 4 + .../Operations/OpDefLibrary.cs | 49 + .../Operations/Operation.Output.cs | 2 +- .../Operations/Operation.cs | 5 +- .../Operations/_EagerTensorArray.cs | 6 +- .../Operations/_GraphTensorArray.cs | 179 ++- .../Operations/array_ops.cs | 24 + .../Operations/control_flow_ops.cs | 9 +- .../Operations/control_flow_util.py.cs | 77 ++ .../Operations/gen_functional_ops.cs | 1066 ++++++++++++-- .../Operations/gen_list_ops.cs | 1227 +++++++++++++++++ src/TensorFlowNET.Core/Operations/list_ops.cs | 111 ++ .../Operations/tensor_array_ops.cs | 20 +- src/TensorFlowNET.Core/Operations/while_v2.cs | 401 ++++++ .../Tensors/Tensor.Creation.cs | 7 + src/TensorFlowNET.Core/Tensors/TensorArray.cs | 24 + src/TensorFlowNET.Core/Tensors/Tensors.cs | 54 +- src/TensorFlowNET.Core/ops.cs | 2 +- src/TensorFlowNET.Keras/BackendImpl.cs | 95 +- src/TensorFlowNET.Keras/Engine/Model.Build.cs | 2 +- .../Engine/Model.Evaluate.cs | 4 +- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.Train.cs | 2 +- .../Layers/Rnn/DropoutRNNCellMixin.cs | 11 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 39 +- .../Layers/Rnn/RnnCellBase.cs | 24 - .../Layers/Rnn/SimpleRNNCell.cs | 7 +- .../Layers/Rnn/StackedRNNCells.cs | 152 +- src/TensorFlowNET.Keras/Utils/RnnUtils.cs | 35 +- .../ManagedAPI/ControlFlowApiTest.cs | 4 +- tools/Tensorflow.CodeGen/FunctionGenerator.cs | 24 +- tools/Tensorflow.CodeGen/Program.cs | 2 +- tools/Tensorflow.CodeGen/Utils.cs | 8 +- 53 files changed, 3662 insertions(+), 507 deletions(-) create mode 100644 src/TensorFlowNET.Core/Common/Types/FakeTensorByTensorArray.cs rename src/TensorFlowNET.Core/Common/Types/{INest.cs => INestStructure.cs} (65%) create mode 100644 src/TensorFlowNET.Core/Framework/auto_control_deps_utils.cs create mode 100644 src/TensorFlowNET.Core/Operations/gen_list_ops.cs create mode 100644 src/TensorFlowNET.Core/Operations/list_ops.cs create mode 100644 src/TensorFlowNET.Core/Operations/while_v2.cs delete mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs diff --git a/src/TensorFlowNET.Core/APIs/c_api.cs b/src/TensorFlowNET.Core/APIs/c_api.cs index 10f678e0..6049c95c 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.cs @@ -16,6 +16,7 @@ using System; using System.Runtime.InteropServices; +using static Tensorflow.CppShapeInferenceResult.Types; namespace Tensorflow { @@ -50,6 +51,19 @@ namespace Tensorflow return handle == IntPtr.Zero ? String.Empty : Marshal.PtrToStringAnsi(handle); } + public unsafe static byte[] ByteStringPiece(IntPtr handle) + { + byte* str_data = (byte*)handle.ToPointer(); + List bytes = new List(); + byte current = 255; + while (current != ((byte)'\0')) + { + current = *(str_data++); + bytes.Add(current); + } + return bytes.Take(bytes.Count - 1).ToArray(); + } + [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate void Deallocator(IntPtr data, IntPtr size, ref DeallocatorArgs args); diff --git a/src/TensorFlowNET.Core/APIs/tf.control_flow.cs b/src/TensorFlowNET.Core/APIs/tf.control_flow.cs index 239487e0..cd5a71e5 100644 --- a/src/TensorFlowNET.Core/APIs/tf.control_flow.cs +++ b/src/TensorFlowNET.Core/APIs/tf.control_flow.cs @@ -46,10 +46,10 @@ namespace Tensorflow Tensor loop_vars, int parallel_iterations = 10) { - Func cond1 = x + Func cond1 = x => cond(x[0]); - Func body1 = x + Func body1 = x => new[] { body(x[0]) }; var results = control_flow_ops.while_loop(cond1, @@ -58,9 +58,9 @@ namespace Tensorflow return results[0]; } - public Tensor[] while_loop(Func cond, - Func body, - Tensor[] loop_vars, + public Tensor[] while_loop(Func cond, + Func body, + Tensors loop_vars, int parallel_iterations = 10, string name = null) => control_flow_ops.while_loop(cond, body, loop_vars, diff --git a/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs index 6cf62e7b..287b48cc 100644 --- a/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs +++ b/src/TensorFlowNET.Core/Common/Extensions/LinqExtensions.cs @@ -18,7 +18,12 @@ namespace Tensorflow.Common.Extensions return sequence.Take(sequence.Count() - count); } #endif - public static Tensors ToTensors(this IEnumerable tensors) + public static Tensors ToTensors(this Tensor[] tensors) + { + return new Tensors(tensors); + } + + public static Tensors ToTensors(this IList tensors) { return new Tensors(tensors); } diff --git a/src/TensorFlowNET.Core/Common/Types/FakeTensorByTensorArray.cs b/src/TensorFlowNET.Core/Common/Types/FakeTensorByTensorArray.cs new file mode 100644 index 00000000..d0c35ee7 --- /dev/null +++ b/src/TensorFlowNET.Core/Common/Types/FakeTensorByTensorArray.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Common.Types +{ + /// + /// This is a temp solution, which should be removed after refactoring `Tensors` + /// + [Obsolete] + public class FakeTensorByTensorArray: Tensor + { + public TensorArray TensorArray { get; set; } + + public FakeTensorByTensorArray(TensorArray array) + { + TensorArray = array; + } + } +} diff --git a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs index c61d04b2..40190315 100644 --- a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs +++ b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs @@ -5,136 +5,80 @@ using System.Text; namespace Tensorflow.Common.Types { - public class GeneralizedTensorShape: IEnumerable, INestStructure, INestable + public class GeneralizedTensorShape: Nest { - public TensorShapeConfig[] Shapes { get; set; } - /// - /// create a single-dim generalized Tensor shape. - /// - /// - public GeneralizedTensorShape(int dim, int size = 1) - { - var elem = new TensorShapeConfig() { Items = new long?[] { dim } }; - Shapes = Enumerable.Repeat(elem, size).ToArray(); - //Shapes = new TensorShapeConfig[size]; - //Shapes.Initialize(new TensorShapeConfig() { Items = new long?[] { dim } }); - //Array.Initialize(Shapes, new TensorShapeConfig() { Items = new long?[] { dim } }); - ////Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; - } + ////public TensorShapeConfig[] Shapes { get; set; } + ///// + ///// create a single-dim generalized Tensor shape. + ///// + ///// + //public GeneralizedTensorShape(int dim, int size = 1) + //{ + // var elem = new TensorShapeConfig() { Items = new long?[] { dim } }; + // Shapes = Enumerable.Repeat(elem, size).ToArray(); + // //Shapes = new TensorShapeConfig[size]; + // //Shapes.Initialize(new TensorShapeConfig() { Items = new long?[] { dim } }); + // //Array.Initialize(Shapes, new TensorShapeConfig() { Items = new long?[] { dim } }); + // ////Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; + //} - public GeneralizedTensorShape(Shape shape) + public GeneralizedTensorShape(Shape value, string? name = null) { - Shapes = new TensorShapeConfig[] { shape }; + NodeValue = value; + NestType = NestType.Node; } - public GeneralizedTensorShape(TensorShapeConfig shape) + public GeneralizedTensorShape(IEnumerable values, string? name = null) { - Shapes = new TensorShapeConfig[] { shape }; + ListValue = values.Select(s => new Nest(s) as INestStructure).ToList(); + Name = name; + NestType = NestType.List; } - public GeneralizedTensorShape(TensorShapeConfig[] shapes) + public GeneralizedTensorShape(Dictionary value, string? name = null) { - Shapes = shapes; + DictValue = value.ToDictionary(x => x.Key, x => new Nest(x.Value) as INestStructure); + Name = name; + NestType = NestType.Dictionary; } - public GeneralizedTensorShape(IEnumerable shape) + public GeneralizedTensorShape(Nest other) { - Shapes = shape.Select(x => (TensorShapeConfig)x).ToArray(); + NestType = other.NestType; + NodeValue = other.NodeValue; + DictValue = other.DictValue; + ListValue = other.ListValue; + Name = other.Name; } public Shape ToSingleShape() { - if (Shapes.Length != 1) + var shapes = Flatten().ToList(); + if (shapes.Count != 1) { throw new ValueError("The generalized shape contains more than 1 dim."); } - var shape_config = Shapes[0]; - Debug.Assert(shape_config is not null); - return new Shape(shape_config.Items.Select(x => x is null ? -1 : x.Value).ToArray()); + return shapes[0]; } public long ToNumber() { - if(Shapes.Length != 1 || Shapes[0].Items.Length != 1) + var shapes = Flatten().ToList(); + if (shapes.Count != 1 || shapes[0].ndim != 1) { throw new ValueError("The generalized shape contains more than 1 dim."); } - var res = Shapes[0].Items[0]; - return res is null ? -1 : res.Value; - } - - public Shape[] ToShapeArray() - { - return Shapes.Select(x => new Shape(x.Items.Select(y => y is null ? -1 : y.Value).ToArray())).ToArray(); - } - - public IEnumerable Flatten() - { - List result = new List(); - foreach(var shapeConfig in Shapes) - { - result.AddRange(shapeConfig.Items); - } - return result; - } - public INestStructure MapStructure(Func func) - { - List> lists = new(); - foreach(var shapeConfig in Shapes) - { - lists.Add(new Nest(shapeConfig.Items.Select(x => new Nest(func(x))))); - } - return new Nest(lists); - } - - public Nest AsNest() - { - Nest DealWithSingleShape(TensorShapeConfig config) - { - if (config.Items.Length == 0) - { - return Nest.Empty; - } - else if (config.Items.Length == 1) - { - return new Nest(config.Items[0]); - } - else - { - return new Nest(config.Items.Select(x => new Nest(x))); - } - } - - if(Shapes.Length == 0) - { - return Nest.Empty; - } - else if(Shapes.Length == 1) - { - return DealWithSingleShape(Shapes[0]); - } - else - { - return new Nest(Shapes.Select(s => DealWithSingleShape(s))); - } + return shapes[0].dims[0]; } - - - public static implicit operator GeneralizedTensorShape(int dims) - => new GeneralizedTensorShape(dims); - - public IEnumerator GetEnumerator() + public INestStructure ToTensorShapeConfigs() { - foreach (var shape in Shapes) - { - yield return shape.Items; - } + return MapStructure(s => new TensorShapeConfig() { Items = s.dims.Select(x => x == -1 ? null : x).ToArray() }); } - IEnumerator IEnumerable.GetEnumerator() + public static implicit operator GeneralizedTensorShape(Shape shape) { - return GetEnumerator(); + return new GeneralizedTensorShape(shape); } } } diff --git a/src/TensorFlowNET.Core/Common/Types/INest.cs b/src/TensorFlowNET.Core/Common/Types/INestStructure.cs similarity index 65% rename from src/TensorFlowNET.Core/Common/Types/INest.cs rename to src/TensorFlowNET.Core/Common/Types/INestStructure.cs index 001141dd..32b66293 100644 --- a/src/TensorFlowNET.Core/Common/Types/INest.cs +++ b/src/TensorFlowNET.Core/Common/Types/INestStructure.cs @@ -10,6 +10,19 @@ namespace Tensorflow.Common.Types /// public interface INestStructure: INestable { + NestType NestType { get; } + + /// + /// The item count of depth 1 of the nested structure. + /// For example, [1, 2, [3, 4, 5]] has ShallowNestedCount = 3. + /// + int ShallowNestedCount { get; } + /// + /// The total item count of depth 1 of the nested structure. + /// For example, [1, 2, [3, 4, 5]] has TotalNestedCount = 5. + /// + int TotalNestedCount { get; } + /// /// Flatten the Nestable object. Node that if the object contains only one value, /// it will be flattened to an enumerable with one element. diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs b/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs index b67d11f4..dc7fd3a1 100644 --- a/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs +++ b/src/TensorFlowNET.Core/Common/Types/Nest.Static.cs @@ -13,7 +13,7 @@ namespace Tensorflow.Common.Types /// /// /// - public static Nest PackSequenceAs(INestable template, T[] flatItems) + public static Nest PackSequenceAs(INestable template, TOut[] flatItems) { return template.AsNest().PackSequence(flatItems); } diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.cs b/src/TensorFlowNET.Core/Common/Types/Nest.cs index 84a60402..4de7d1fa 100644 --- a/src/TensorFlowNET.Core/Common/Types/Nest.cs +++ b/src/TensorFlowNET.Core/Common/Types/Nest.cs @@ -28,27 +28,58 @@ namespace Tensorflow.Common.Types public static Nest Empty => _empty; public NestType NestType { get; protected set; } public string? Name { get; set; } - public T? Value { get; protected set; } - public List>? ListValue { get; protected set; } - public Dictionary>? DictValue { get; protected set; } + public T? NodeValue { get; protected set; } + public List>? ListValue { get; protected set; } + public Dictionary>? DictValue { get; protected set; } + + public int ShallowNestedCount + { + get + { + if (NestType == NestType.Empty) + { + return 0; + } + else if (NestType == NestType.Node) + { + return 1; + } + else if (NestType == NestType.List) + { + return ListValue!.Count; + } + else // dict + { + return DictValue!.Count; + } + } + } + + public int TotalNestedCount + { + get + { + return Flatten().Count(); + } + } protected Nest() { } public Nest(T value, string? name = null) { - Value = value; + NodeValue = value; Name = name; NestType = NestType.Node; } - public Nest(IEnumerable> values, string? name = null) + public Nest(IEnumerable> values, string? name = null) { ListValue = values.ToList(); Name = name; NestType = NestType.List; } - public Nest(Dictionary> value, string? name = null) + public Nest(Dictionary> value, string? name = null) { DictValue = value; Name = name; @@ -58,7 +89,7 @@ namespace Tensorflow.Common.Types public Nest(Nest other) { NestType = other.NestType; - Value = other.Value; + NodeValue = other.NodeValue; DictValue = other.DictValue; ListValue = other.ListValue; Name = other.Name; @@ -78,17 +109,17 @@ namespace Tensorflow.Common.Types /// /// /// - public virtual Nest PackSequence(T[] flatItems) + public virtual Nest PackSequence(TOut[] flatItems) { if(flatItems.Length == 0) { - return Nest.Empty; + return Nest.Empty; } int index = 0; return PackSequenceInternal(this, flatItems, ref index); } - private static Nest PackSequenceInternal(Nest template, T[] flatItems, ref int index) + private static Nest PackSequenceInternal(Nest template, TOut[] flatItems, ref int index) { if(template.NestType == NestType.Node) { @@ -96,25 +127,25 @@ namespace Tensorflow.Common.Types { throw new InvalidArgumentError("The template and flat items are not matched."); } - return new Nest(flatItems[index++]); + return new Nest(flatItems[index++]); } else if(template.NestType == NestType.List) { - List> nestedObjects = new List>(); + List> nestedObjects = new List>(); for (int i = 0; i < template.ListValue!.Count; i++) { - nestedObjects.Add(PackSequenceInternal(template.ListValue![i], flatItems, ref index)); + nestedObjects.Add(PackSequenceInternal(template.ListValue![i].AsNest(), flatItems, ref index)); } - return new Nest(nestedObjects); + return new Nest(nestedObjects); } else if(template.NestType == NestType.Node) { - Dictionary> dict = new Dictionary>(); + Dictionary> dict = new Dictionary>(); foreach(var (key, value) in template.DictValue!) { - dict[key] = PackSequenceInternal(value, flatItems, ref index); + dict[key] = PackSequenceInternal(value.AsNest(), flatItems, ref index); } - return new Nest(dict); + return new Nest(dict); } // Consider Empty as invalid type. throw new InvalidArgumentError("When using `PackSequenceAs`, the template cannot contain empty node."); @@ -223,10 +254,10 @@ namespace Tensorflow.Common.Types public static Nest ReduceFrom(INestStructure input) where TOut: INestStructure { var nested = input.AsNest(); - return ReduceInternal(nested); + return ReduceInternal(nested).AsNest(); } - private static Nest ReduceInternal(Nest node) where TOut : INestStructure + private static INestStructure ReduceInternal(Nest node) where TOut : INestStructure { if(node.NestType == NestType.Empty) { @@ -234,15 +265,15 @@ namespace Tensorflow.Common.Types } else if(node.NestType == NestType.Node) { - return node.Value!.AsNest(); + return node.NodeValue!.AsNest(); } else if(node.NestType == NestType.List) { - return new Nest(node.ListValue!.Select(x => ReduceInternal(x))); + return new Nest(node.ListValue!.Select(x => ReduceInternal(x.AsNest()))); } else // Dictionary type { - return new Nest(node.DictValue!.ToDictionary(x => x.Key, x => ReduceInternal(x.Value))); + return new Nest(node.DictValue!.ToDictionary(x => x.Key, x => ReduceInternal(x.Value.AsNest()))); } } @@ -252,7 +283,7 @@ namespace Tensorflow.Common.Types { if(index == 0) { - result = node.Value!; + result = node.NodeValue!; return true; } result = default(T); @@ -264,7 +295,7 @@ namespace Tensorflow.Common.Types { if(index == 0) { - return FindInternal(item, index, out result); + return FindInternal(item.AsNest(), index, out result); } index--; } @@ -277,7 +308,7 @@ namespace Tensorflow.Common.Types { if (index == 0) { - return FindInternal(item, index, out result); + return FindInternal(item.AsNest(), index, out result); } index--; } @@ -297,7 +328,7 @@ namespace Tensorflow.Common.Types { if (index == 0) { - node.Value = newValue; + node.NodeValue = newValue; return true; } return false; @@ -308,7 +339,7 @@ namespace Tensorflow.Common.Types { if (index == 0) { - return SetInternal(item, index, newValue); + return SetInternal(item.AsNest(), index, newValue); } index--; } @@ -320,7 +351,7 @@ namespace Tensorflow.Common.Types { if (index == 0) { - return SetInternal(item, index, newValue); + return SetInternal(item.AsNest(), index, newValue); } index--; } @@ -336,13 +367,13 @@ namespace Tensorflow.Common.Types { if (node.NestType == NestType.Node) { - yield return node.Value!; + yield return node.NodeValue!; } else if (node.NestType == NestType.List) { foreach (var item in node.ListValue!) { - foreach(var val in FlattenInternal(item)) + foreach(var val in FlattenInternal(item.AsNest())) { yield return val; } @@ -352,7 +383,7 @@ namespace Tensorflow.Common.Types { foreach (var item in node.DictValue!.Values) { - foreach (var val in FlattenInternal(item)) + foreach (var val in FlattenInternal(item.AsNest())) { yield return val; } @@ -364,23 +395,23 @@ namespace Tensorflow.Common.Types { if (NestType == NestType.Node) { - return new Nest(func(Value!)); + return new Nest(func(NodeValue!)); } else if (NestType == NestType.List) { List> outs = new List>(); foreach (var item in ListValue!) { - outs.Add(item.MapStructureInternal(func)); + outs.Add(item.AsNest().MapStructureInternal(func)); } return new Nest(outs); } else if (NestType == NestType.Dictionary) { - Dictionary> outs = new Dictionary>(); + Dictionary> outs = new Dictionary>(); foreach (var (key, value) in DictValue!) { - outs.Add(key, value.MapStructureInternal(func)); + outs.Add(key, value.AsNest().MapStructureInternal(func)); } return new Nest(outs); } @@ -417,14 +448,14 @@ namespace Tensorflow.Common.Types } if (node.NestType == NestType.Node) { - sb.Append(node.Value!.ToString()); + sb.Append(node.NodeValue!.ToString()); } else if (node.NestType == NestType.List) { sb.Append("["); for(int i = 0; i < node.ListValue!.Count; i++) { - WriteString(node.ListValue![i], sb); + WriteString(node.ListValue![i].AsNest(), sb); if(i != node.ListValue!.Count - 1) { sb.Append(", "); @@ -440,7 +471,7 @@ namespace Tensorflow.Common.Types foreach (var (key, value) in node.DictValue!) { sb.Append($"{key}: "); - WriteString(value, sb); + WriteString(value.AsNest(), sb); if (i != count - 1) { sb.Append(", "); @@ -454,5 +485,15 @@ namespace Tensorflow.Common.Types sb.Append(""); } } + + public static implicit operator Nest((INestStructure, INestStructure) inputs) + { + return new Nest(new INestStructure[] { inputs.Item1, inputs.Item2 }); + } + + public static implicit operator Nest((INestStructure, INestStructure, INestStructure) inputs) + { + return new Nest(new INestStructure[] { inputs.Item1, inputs.Item2, inputs.Item3 }); + } } } diff --git a/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs b/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs index 554ca526..cf199455 100644 --- a/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs +++ b/src/TensorFlowNET.Core/Common/Types/NestDictionary.cs @@ -6,7 +6,11 @@ namespace Tensorflow.Common.Types { public class NestDictionary : INestStructure, IDictionary where TKey : notnull { + public NestType NestType => NestType.Dictionary; public IDictionary Value { get; set; } + public int ShallowNestedCount => Values.Count; + + public int TotalNestedCount => Values.Count; public NestDictionary(IDictionary dict) { Value = dict; diff --git a/src/TensorFlowNET.Core/Common/Types/NestList.cs b/src/TensorFlowNET.Core/Common/Types/NestList.cs index 08218718..e38675da 100644 --- a/src/TensorFlowNET.Core/Common/Types/NestList.cs +++ b/src/TensorFlowNET.Core/Common/Types/NestList.cs @@ -10,29 +10,34 @@ namespace Tensorflow.Common.Types /// public sealed class NestList : INestStructure, IEnumerable { - public List Value { get; set; } + public NestType NestType => NestType.List; + public List Values { get; set; } + public int ShallowNestedCount => Values.Count; + + public int TotalNestedCount => Values.Count; + public NestList(IEnumerable values) { - Value = new List(values); + Values = new List(values); } public IEnumerable Flatten() { - return Value; + return Values; } public INestStructure MapStructure(Func func) { - return new NestList(Value.Select(x => func(x))); + return new NestList(Values.Select(x => func(x))); } public Nest AsNest() { - return new Nest(Value.Select(x => new Nest(x))); + return new Nest(Values.Select(x => new Nest(x))); } // Enumerator implementation public IEnumerator GetEnumerator() { - return Value.GetEnumerator(); + return Values.GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() diff --git a/src/TensorFlowNET.Core/Common/Types/NestNode.cs b/src/TensorFlowNET.Core/Common/Types/NestNode.cs index 1dad421d..701aade9 100644 --- a/src/TensorFlowNET.Core/Common/Types/NestNode.cs +++ b/src/TensorFlowNET.Core/Common/Types/NestNode.cs @@ -10,7 +10,11 @@ namespace Tensorflow.Common.Types /// public class NestNode : INestStructure { + public NestType NestType => NestType.Node; public T Value { get; set; } + public int ShallowNestedCount => 1; + + public int TotalNestedCount => 1; public NestNode(T value) { Value = value; diff --git a/src/TensorFlowNET.Core/Data/DatasetV2.cs b/src/TensorFlowNET.Core/Data/DatasetV2.cs index 324d7e83..c1762d67 100644 --- a/src/TensorFlowNET.Core/Data/DatasetV2.cs +++ b/src/TensorFlowNET.Core/Data/DatasetV2.cs @@ -161,8 +161,8 @@ namespace Tensorflow break; } - yield return (new Tensors(results.Take(FirstInputTensorCount)), results.Length == FirstInputTensorCount ? - null : new Tensors(results.Skip(FirstInputTensorCount))); + yield return (new Tensors(results.Take(FirstInputTensorCount).ToArray()), results.Length == FirstInputTensorCount ? + null : new Tensors(results.Skip(FirstInputTensorCount).ToArray())); } } diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs index f1a09ed7..5f156fd9 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs @@ -359,6 +359,8 @@ namespace Tensorflow.Eager case TF_AttrType.TF_ATTR_FUNC: if (value is ConcreteFunction func) c_api.TFE_OpSetAttrFunctionName(op, key, func.func_graph.FuncName, func.func_graph.FuncName.Length); + else if(value is string str) + c_api.TFE_OpSetAttrFunctionName(op, key, str, str.Length); else throw new NotImplementedException("TF_AttrType.TF_ATTR_FUNC"); break; diff --git a/src/TensorFlowNET.Core/Framework/Models/TensorSpec.cs b/src/TensorFlowNET.Core/Framework/Models/TensorSpec.cs index 083d4813..ac099ae2 100644 --- a/src/TensorFlowNET.Core/Framework/Models/TensorSpec.cs +++ b/src/TensorFlowNET.Core/Framework/Models/TensorSpec.cs @@ -1,4 +1,5 @@ using System.Linq; +using Tensorflow.Eager; namespace Tensorflow.Framework.Models { @@ -24,5 +25,17 @@ namespace Tensorflow.Framework.Models shapes.Insert(0, dim); return new TensorSpec(shapes.ToArray(), _dtype); } + + public static TensorSpec FromTensor(Tensor tensor, string? name = null) + { + if(tensor is EagerTensor) + { + return new TensorSpec(tensor.shape, tensor.dtype, name); + } + else + { + return new TensorSpec(tensor.shape, tensor.dtype, name ?? tensor.name); + } + } } } diff --git a/src/TensorFlowNET.Core/Framework/auto_control_deps_utils.cs b/src/TensorFlowNET.Core/Framework/auto_control_deps_utils.cs new file mode 100644 index 00000000..28d9e500 --- /dev/null +++ b/src/TensorFlowNET.Core/Framework/auto_control_deps_utils.cs @@ -0,0 +1,89 @@ +using Tensorflow.Graphs; + +namespace Tensorflow.Framework +{ + internal static class auto_control_deps_utils + { + public static readonly string READ_ONLY_RESOURCE_INPUTS_ATTR = "_read_only_resource_inputs"; + public static List get_read_only_resource_input_indices_graph(FuncGraph func_graph) + { + List result = new List(); + // A cache to store the read only resource inputs of an Op. + // Operation -> ObjectIdentitySet of resource handles. + Dictionary> opReadOnlyResourceInputs = + new Dictionary>(); + + for (int inputIndex = 0; inputIndex < func_graph.Inputs.Length; inputIndex++) + { + Tensor t = func_graph.Inputs[inputIndex]; + if (t.dtype != dtypes.resource) + continue; + + bool readOnly = true; + foreach (var op in t.consumers()) + { + if (opReadOnlyResourceInputs.ContainsKey(op)) + { + if (!opReadOnlyResourceInputs[op].Contains(t)) + { + readOnly = false; + break; + } + } + else + { + List indices = _get_read_only_resource_input_indices_op(op); + opReadOnlyResourceInputs[op] = new HashSet( + indices.Select(i => op.inputs[i])); + if (!opReadOnlyResourceInputs[op].Contains(t)) + { + readOnly = false; + break; + } + } + } + + if (readOnly) + result.Add(inputIndex); + } + + return result; + } + + private static List _get_read_only_resource_input_indices_op(Operation op) + { + // ignore the RESOURCE_READ_OPS + + int[] read_only_input_indices; + + try + { + read_only_input_indices = op.get_attr(READ_ONLY_RESOURCE_INPUTS_ATTR); + } + catch (InvalidArgumentError) + { + return new List(); + } + + int read_only_index = 0; + List result = new(); + for (int i = 0; i < op.inputs.Length; i++) + { + if (read_only_index >= read_only_input_indices.Length) + { + break; + } + if (op.inputs[i].dtype != dtypes.resource) + { + continue; + } + if (read_only_index < read_only_input_indices.Length && i == read_only_input_indices[read_only_index]) + { + result.Add(i); + read_only_index++; + } + } + return result; + } + } +} diff --git a/src/TensorFlowNET.Core/Framework/function_def_lib.cs b/src/TensorFlowNET.Core/Framework/function_def_lib.cs index 67f8d324..488c6b65 100644 --- a/src/TensorFlowNET.Core/Framework/function_def_lib.cs +++ b/src/TensorFlowNET.Core/Framework/function_def_lib.cs @@ -42,10 +42,10 @@ namespace Tensorflow.Framework func_graph.as_default(); importer.import_graph_def(graph_def, name: "", validate_colocation_constraints: false); var input_tensor_names = fdef.Signature.InputArg.Select(x => nested_to_flat_tensor_name[x.Name]); - func_graph.Inputs = new Tensors(input_tensor_names.Select(x => func_graph.get_tensor_by_name(x))); + func_graph.Inputs = new Tensors(input_tensor_names.Select(x => func_graph.get_tensor_by_name(x)).ToArray()); var output_tensor_names = fdef.Signature.OutputArg.Select(x => nested_to_flat_tensor_name[fdef.Ret[x.Name]]); - func_graph.Outputs = new Tensors(output_tensor_names.Select(x => func_graph.get_tensor_by_name(x))); + func_graph.Outputs = new Tensors(output_tensor_names.Select(x => func_graph.get_tensor_by_name(x)).ToArray()); // TODO(Rinne): func_graph.ControlOutputs _set_handle_data(func_graph, fdef); diff --git a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs index 88dce7d9..8742e453 100644 --- a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs +++ b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs @@ -8,6 +8,7 @@ using Tensorflow.Gradients; using Tensorflow.Graphs; using Tensorflow.Train; using Tensorflow.Util; +using Tensorflow.Common.Extensions; using static Tensorflow.Binding; namespace Tensorflow.Functions @@ -40,6 +41,18 @@ namespace Tensorflow.Functions public Tensor[] FlatStructuredOutputs => func_graph.FlatStructuredOutputs; public IEnumerable Variables => func_graph.Variables; public IEnumerable TrainableVariables => func_graph.TrainableVariables; + internal NameAttrList AsNameAttrList + { + get + { + NameAttrList ret = new() { Name = this.Name }; + foreach (var (name, value) in _attrs) + { + ret.Attr[name] = value; + } + return ret; + } + } public ConcreteFunction(string name) { diff --git a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs index 3bce52ea..ba7d7068 100644 --- a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs +++ b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs @@ -81,7 +81,7 @@ public class FuncGraph : Graph, IDisposable public IEnumerable TrainableVariables => Variables.Where(v => v.Trainable); public Dictionary Attrs { get; set; } - Dictionary _captures + internal Dictionary _captures = new Dictionary(); public Tensor[] external_captures @@ -399,7 +399,7 @@ public class FuncGraph : Graph, IDisposable var flat_func_args = nest.flatten(func_args as object); var flat_func_kwargs = nest.flatten(func_kwargs as object); func_graph.Inputs = new Tensors(flat_func_args.concat(flat_func_kwargs) - .Where(x => x is Tensor).Select(x => (Tensor)x)); + .Where(x => x is Tensor).Select(x => (Tensor)x).ToArray()); //var func_args_before = nest.pack_sequence_as(func_args, flat_func_args, true); //var func_kwargs_before = nest.pack_sequence_as(func_kwargs, flat_func_kwargs, true); diff --git a/src/TensorFlowNET.Core/Graphs/Graph.cs b/src/TensorFlowNET.Core/Graphs/Graph.cs index eb8df581..9e879a0f 100644 --- a/src/TensorFlowNET.Core/Graphs/Graph.cs +++ b/src/TensorFlowNET.Core/Graphs/Graph.cs @@ -129,7 +129,7 @@ namespace Tensorflow } } - protected Graph outer_graph; + internal Graph outer_graph; public Graph OuterGraph => outer_graph; public Dictionary Functions => _functions; public SafeGraphHandle c_graph => _handle; diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs index d12ed1ad..8614391a 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -7,13 +7,19 @@ namespace Tensorflow.Keras.Layers.Rnn { public interface IRnnCell: ILayer { - GeneralizedTensorShape StateSize { get; } - GeneralizedTensorShape OutputSize { get; } - bool IsTFRnnCell { get; } + /// + /// If the derived class tends to not implement it, please return null. + /// + GeneralizedTensorShape? StateSize { get; } + /// + /// If the derived class tends to not implement it, please return null. + /// + GeneralizedTensorShape? OutputSize { get; } /// /// Whether the optional RNN args are supported when appying the layer. /// In other words, whether `Apply` is overwrited with process of `RnnOptionalArgs`. /// bool SupportOptionalArgs { get; } + Tensors GetInitialState(Tensors inputs, Tensor batch_size, TF_DataType dtype); } } diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 26646b76..b651089a 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -181,6 +181,10 @@ namespace Tensorflow { throw new NotImplementedException(); } + public Tensors GetInitialState(Tensors inputs = null, Tensor batch_size = null, TF_DataType dtype = TF_DataType.DtInvalid) + { + throw new NotImplementedException(); + } public GeneralizedTensorShape StateSize => throw new NotImplementedException(); public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); public bool IsTFRnnCell => throw new NotImplementedException(); diff --git a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs index 76a222ba..5ff5ccff 100644 --- a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs +++ b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs @@ -15,9 +15,11 @@ ******************************************************************************/ using Google.Protobuf; +using Google.Protobuf.Collections; using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Functions; using static Tensorflow.Binding; using static Tensorflow.OpDef.Types; @@ -420,6 +422,12 @@ namespace Tensorflow case "list(shape)": attr_value.List.Shape.AddRange((value as Shape[]).Select(x => _MakeShape(x, attr_def))); break; + case "func": + attr_value.Func = _MakeFunc(value, attr_def.Name); + break; + case "list(func)": + attr_value.List.Func.AddRange(_MakeFuncList(value, attr_def.Name)); + break; default: throw new TypeError($"SetAttrValue: can't not convert attr_def.Type '{attr_def.Type}' to protos."); } @@ -427,6 +435,47 @@ namespace Tensorflow return attr_value; } + private NameAttrList _MakeFunc(object func, string arg_name) + { + if(func is NameAttrList attrList) + { + return attrList; + } + NameAttrList fn_attr; + if(func is string funcStr) + { + fn_attr = new NameAttrList() { Name = funcStr }; + } + else if(func is ConcreteFunction concrete) + { + concrete.AddTograph(ops.get_default_graph()); + fn_attr = concrete.AsNameAttrList; + } + else if(func is EagerDefinedFunction eager) + { + eager.AddToGraph(ops.get_default_graph()); + fn_attr = new NameAttrList() { Name = eager.Name }; + } + else + { + throw new TypeError($"Don't know how to convert {func} to a func for argument {arg_name}"); + } + return fn_attr; + } + + private List _MakeFuncList(object funcList, string arg_name) + { + List res = new List(); + if(funcList is IEnumerable enumerable) + { + foreach(var func in enumerable) + { + res.Add(_MakeFunc(func, arg_name)); + } + } + return res; + } + private bool _IsListParameter(ArgDef arg) { if (!String.IsNullOrEmpty(arg.NumberAttr)) diff --git a/src/TensorFlowNET.Core/Operations/Operation.Output.cs b/src/TensorFlowNET.Core/Operations/Operation.Output.cs index 2955a13f..2329a478 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.Output.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.Output.cs @@ -34,7 +34,7 @@ namespace Tensorflow return num; } - protected Tensor[] _outputs; + internal Tensor[] _outputs; public virtual Tensor[] outputs => _outputs; public Tensor output => _outputs.FirstOrDefault(); diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs index a789c5f4..5e689c65 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.cs @@ -46,9 +46,9 @@ namespace Tensorflow /// public partial class Operation : ITensorOrOperation { - private readonly IntPtr _handle; // _c_op in python + protected IntPtr _handle; // _c_op in python - private readonly Graph _graph; + protected Graph _graph; internal Func _gradient_function; @@ -69,6 +69,7 @@ namespace Tensorflow //private OperationDescription _op_desc; public NodeDef node_def => GetNodeDef(); + protected Operation() { } public Operation(IntPtr handle, Graph g = null) { diff --git a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs index 08e73fe6..59176060 100644 --- a/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_EagerTensorArray.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Eager; using Tensorflow.Framework; using static Tensorflow.Binding; @@ -38,10 +39,6 @@ namespace Tensorflow.Operations bool _infer_shape; public override bool infer_shape => _infer_shape; - public bool _dynamic_size; - public Shape _element_shape; - - public List _colocate_with; Tensor _handle; public override Tensor handle => _handle; @@ -56,6 +53,7 @@ namespace Tensorflow.Operations bool infer_shape = true, Shape? element_shape = null, bool colocate_with_first_write_call = true, string name = null) { + _size = size; _flow = constant_op.constant(0); _infer_shape = infer_shape; _element_shape = element_shape ?? Shape.Null; diff --git a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs index dde2624a..4c3fde31 100644 --- a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs @@ -16,7 +16,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Tensorflow.Common.Types; using Tensorflow.Eager; using static Tensorflow.Binding; @@ -33,18 +35,18 @@ namespace Tensorflow.Operations /// first tensor written to it. /// bool _colocate_with_first_write_call; - public bool colocate_with_first_write_call => _colocate_with_first_write_call; + public override bool colocate_with_first_write_call => _colocate_with_first_write_call; bool _infer_shape; - public bool infer_shape => _infer_shape; - public bool _dynamic_size; + public override bool infer_shape => _infer_shape; public List _element_shape; public List _colocate_with; internal Tensor _handle; - public Tensor handle => _handle; + public override Tensor handle => _handle; internal Tensor _flow; + public override Tensor flow => _flow; public _GraphTensorArray(TF_DataType dtype, Tensor size, bool? dynamic_size = null, bool? clear_after_read = null, string tensor_array_name = null, Tensor handle = null, Tensor flow = null, @@ -55,6 +57,7 @@ namespace Tensorflow.Operations dynamic_size = dynamic_size ?? false; _dynamic_size = dynamic_size.Value; _dtype = dtype; + _size = size; _colocate_with_first_write_call = colocate_with_first_write_call; if (colocate_with_first_write_call) @@ -235,4 +238,172 @@ namespace Tensorflow.Operations return value; } } + + public class _GraphTensorArrayV2 : TensorArray + { + internal TF_DataType _dtype; + public override TF_DataType dtype => _dtype; + + /// + /// Used to keep track of what tensors the TensorArray should be + /// colocated with. We choose to colocate the TensorArray with the + /// first tensor written to it. + /// + bool _colocate_with_first_write_call; + public override bool colocate_with_first_write_call => _colocate_with_first_write_call; + + bool _infer_shape; + public override bool infer_shape => _infer_shape; + public Shape _element_shape; + + public List _colocate_with; + + internal Tensor _handle; + public override Tensor handle => _handle; + internal Tensor _flow; + public override Tensor flow => _flow; + + public _GraphTensorArrayV2(TF_DataType dtype, Tensor size, bool? dynamic_size = null, + bool? clear_after_read = null, string tensor_array_name = null, Tensor handle = null, Tensor flow = null, + bool infer_shape = true, Shape? element_shape = null, + bool colocate_with_first_write_call = true, string name = null) + { + Debug.Assert(handle is null); + dynamic_size = dynamic_size ?? false; + _dynamic_size = dynamic_size.Value; + _size = size; + + if(flow is not null && flow.dtype != dtypes.variant) + { + throw new TypeError($"Expected `flow` to be a variant tensor, but received `{flow.dtype}` instead"); + } + if(flow is null && size is null) + { + throw new ValueError("Argument `size` must be provided if argument `flow` is not provided."); + } + if(flow is not null && size is not null) + { + throw new ValueError("Cannot provide both `flow` and `size` arguments at the same time."); + } + if(flow is not null && element_shape is not null) + { + throw new ValueError("Cannot provide both `flow` and `element_shape` arguments at the same time."); + } + + _dtype = dtype; + + _element_shape = element_shape; + _infer_shape = infer_shape; + tf_with(ops.name_scope(name, "TensorArrayV2", new object[] { size, flow }), scope => + { + if (flow is null) + { + _flow = list_ops.tensor_list_reserve(element_shape, size, dtype, scope.scope_name); + } + else + { + _flow = flow; + } + }); + + _colocate_with_first_write_call = false; + _colocate_with = null; + } + + public override TensorArray unstack(Tensor value, string name = null) + { + return tf_with(ops.name_scope(name, "TensorArrayUnstack", new { _flow, value }), delegate + { + value = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); + Debug.Assert(value.dtype == _dtype); + var flow_out = list_ops.tensor_list_from_tensor(value, value.shape.dims.Skip(1).ToArray()); + return tensor_array_ops.build_ta_with_new_flow(this, flow_out); + }); + } + + public TensorArray scatter(Tensor indices, Tensor value, string name = null) + { + return tf_with(ops.name_scope(name, "TensorArrayScatter", new { _flow, value, indices }), delegate + { + value = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); + Debug.Assert(value.dtype == _dtype); + var flow_out = list_ops.tensor_list_scatter(value, indices, _element_shape, _flow); + return tensor_array_ops.build_ta_with_new_flow(this, flow_out); + }); + } + + public override Tensor read(T index, string name = null) + { + if(index is Tensor tensor) + { + return read(tensor, name); + } + else + { + throw new TypeError("Please use non-generic method instead."); + } + } + + public Tensor read(Tensor index, string name = null) + { + return tf_with(tf.name_scope(name, "TensorArrayV2Read", new object[] { _flow, index }), scope => + { + return list_ops.tensor_list_get_item(_flow, index, _dtype, _element_shape, name); + }); + } + + public override TensorArray write(Tensor index, Tensor value, string name = null) + { + return tf_with(ops.name_scope(name, "TensorArrayV2Write", new { _flow, index, value }), delegate + { + value = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); + Debug.Assert(value.dtype == _dtype); + var flow_out = list_ops.tensor_list_set_item(_flow, index, value, _dynamic_size, name); + + return tensor_array_ops.build_ta_with_new_flow(this, flow_out); + }); + } + + public override TensorArray write(int index, T value, string name = null) + { + var value_tensor = ops.convert_to_tensor(value, preferred_dtype: _dtype, name: "value"); + var index_tensor = ops.convert_to_tensor(index, name: "index"); + return write(index_tensor, value_tensor); + } + + private Tensor size(string name = null) + { + if(!_dynamic_size && _size is not null) + { + return ops.convert_to_tensor(_size, dtypes.int32); + } + else + { + return gen_list_ops.tensor_list_length(_flow, name); + } + } + + public override Tensor stack(string name = null) + { + return tf_with(ops.name_scope(name, "TensorArrayV2Stack", _flow), delegate + { + int ta_size; + if(!_dynamic_size && (_size is not null)) + { + ta_size = (int)tensor_util.constant_value(_size); + } + else + { + ta_size = -1; + } + var value = list_ops.tensor_list_stack(_flow, _dtype, ta_size, _element_shape); + return value; + }); + } + + public override Tensor gather(Tensor indices, string name = null) + { + return list_ops.tensor_list_gather(_flow, indices, _dtype, _element_shape, name); + } + } } diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index a0b47aac..ca9e5fae 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -119,6 +119,27 @@ namespace Tensorflow } } + public static Tensor zeros(Tensors shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) + { + dtype = dtype.as_base_dtype(); + Tensor shapeTensor; + if(shape.Length > 1) + { + shapeTensor = ops.convert_to_tensor(shape, dtypes.int32); + if(shapeTensor.ndim > 1) + { + shapeTensor = array_ops.reshape(shapeTensor, new Shape(-1)); + } + } + else + { + shapeTensor = shape[0]; + } + var output = fill(shapeTensor, array_ops.constant(0, dtype), name); + Debug.Assert(output.dtype.as_base_dtype() == dtype); + return output; + } + public static Tensor boolean_mask(T1 tensor, T2 mask, string name = "boolean_mask", int axis = 0) { return tf_with(ops.name_scope(name, values: new { tensor, mask }), delegate @@ -307,6 +328,9 @@ namespace Tensorflow public static Tensor fill(Shape dims, T value, string name = null) => gen_array_ops.fill(dims, ops.convert_to_tensor(value), name: name); + public static Tensor fill(Tensor dims, T value, string name = null) + => gen_array_ops.fill(dims, ops.convert_to_tensor(value), name: name); + /// /// Returns the rank of a tensor. /// diff --git a/src/TensorFlowNET.Core/Operations/control_flow_ops.cs b/src/TensorFlowNET.Core/Operations/control_flow_ops.cs index 862b636f..efd9aba3 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_ops.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_ops.cs @@ -675,16 +675,17 @@ namespace Tensorflow } } - public static Tensor[] while_loop(Func cond, - Func body, - Tensor[] loop_vars, + public static Tensors while_loop(Func cond, + Func body, + Tensors loop_vars, int parallel_iterations = 10, string name = null) { var executing_eagerly = tf.Context.executing_eagerly(); if (!executing_eagerly) { - throw new NotImplementedException(""); + return while_v2.while_loop(cond, body, loop_vars, parallel_iterations: parallel_iterations, + name: name); } return tf_with(ops.name_scope("name", "while"), delegate diff --git a/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs b/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs index c8891119..536d4e3c 100644 --- a/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs +++ b/src/TensorFlowNET.Core/Operations/control_flow_util.py.cs @@ -16,12 +16,20 @@ using System; using System.Linq; +using Tensorflow.Functions; +using Tensorflow.Graphs; using Tensorflow.Operations; +using static Tensorflow.Binding; namespace Tensorflow { public class control_flow_util { + public static readonly bool ENABLE_CONTROL_FLOW_V2 = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_ENABLE_CONTROL_FLOW_V2")) && Environment.GetEnvironmentVariable("TF_ENABLE_CONTROL_FLOW_V2") != "0" || + (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_ENABLE_CONTROL_FLOW_V2")) && Environment.GetEnvironmentVariable("TF_ENABLE_CONTROL_FLOW_V2") != "0") || + (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_ENABLE_COND_V2")) && Environment.GetEnvironmentVariable("TF_ENABLE_COND_V2") != "0") || + (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_ENABLE_WHILE_V2")) && Environment.GetEnvironmentVariable("TF_ENABLE_WHILE_V2") != "0") || + (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_ENABLE_TENSOR_ARRAY_V2")) && Environment.GetEnvironmentVariable("TF_ENABLE_TENSOR_ARRAY_V2") != "0"); /// /// Return true if `op` is an Exit. /// @@ -196,5 +204,74 @@ namespace Tensorflow } return null; } + + public static bool EnableControlFlowV2(Graph graph) + { + return ENABLE_CONTROL_FLOW_V2 || graph.building_function && (graph is not FuncGraph func || func.captures.Length == 0); + + } + + public static string create_new_tf_function(FuncGraph func_graph) + { + var func = new EagerDefinedFunction(func_graph.Name, func_graph, func_graph.Inputs, func_graph.Outputs, new Dictionary()); + func.AddToGraph(func_graph); + return func_graph.Name; + } + + public static (Operation, Tensor[]) get_op_and_outputs(Tensor[] inputs) + { + if(inputs.Length == 0) + { + return (null, new Tensor[0]); + } + else + { + return (inputs[0], inputs); + } + } + + public static Tensor[] run_as_function_for_tape_gradients(Func make_op, Tensor[] inputs) + { + if(gradients_util.PossibleTapeGradientTypes(inputs) == gradients_util.POSSIBLE_GRADIENT_TYPES_HIGHER_ORDER + && !(ops.get_default_graph().building_function)) + { + throw new NotImplementedException(); + } + else + { + return make_op(inputs); + } + } + + public static string unique_fn_name(string scope, string name) + { + return $"{scope}{name}_{ops.uid()}".Replace("/", "_"); + } + + public static bool output_all_intermediates() + { + if (in_defun()) + { + return false; + } + if(tf.Context.FunctionCallOptions.ExecutorType == "SINGLE_THREADED_EXECUTOR") + { + return false; + } + // TODO(Rinne): check this after refactoring keras building. + return false; + } + + public static bool in_defun() + { + if (tf.Context.executing_eagerly()) + { + return false; + } + + var graph = ops.get_default_graph(); + // TODO(Rinne): CondBranchFuncGraph, WhileBodyFuncGraph, WhileCondFuncGraph + return graph is FuncGraph; + } } } diff --git a/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs b/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs index 5663f9c9..e1cf1c13 100644 --- a/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs @@ -1,128 +1,1032 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Xml.Linq; -using Tensorflow.Contexts; +/*Wrappers around TensorFlow ops. This file is MACHINE GENERATED! Do not edit.*/ + using Tensorflow.Eager; -using Tensorflow.Functions; +using Tensorflow.Contexts; using static Tensorflow.Binding; -namespace Tensorflow.Operations +namespace Tensorflow; + +public static class gen_functional_ops { - public class gen_functional_ops + /// + /// An n-way switch statement which calls a single branch function. + /// + /// + /// + /// An n-way switch statement, implementing the following: + /// ``` + /// switch (branch_index) { + /// case 0: + /// output = branches[0](input); + /// break; + /// case 1: + /// output = branches[1](input); + /// break; + /// ... + /// case [[nbranches-1]]: + /// default: + /// output = branches[nbranches-1](input); + /// break; + /// } + /// ``` + /// + /// + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A list of functions each of which takes 'inputs' and returns a list of + /// tensors, whose types are the same as what every other branch returns. + /// + /// + /// + /// + public static Tensor[] _case(Tensor branch_index, Tensors input, TF_DataType[] Tout, object[] branches, Shape[] output_shapes, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Case", name) { args = new object[] { branch_index, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["branches"] = branches, ["output_shapes"] = output_shapes } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return case_eager_fallback(branch_index, input, Tout: Tout, branches: branches, output_shapes: output_shapes, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["branch_index"] = branch_index; + keywords["input"] = input; + keywords["Tout"] = Tout; + keywords["branches"] = branches; + keywords["output_shapes"] = output_shapes; + var _op = tf.OpDefLib._apply_op_helper("Case", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "branches", _op.get_attr("branches"), "output_shapes", _op.get_attr("output_shapes") }; + _execute.record_gradient("Case", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] case_eager_fallback(Tensor branch_index, Tensor input, TF_DataType[] Tout, object[] branches, Shape[] output_shapes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { branch_index, input }; + object[] _attrs = new object[] { "branches", branches, "output_shapes", output_shapes }; + var _result = _execute.execute("Case", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("Case", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// Return the index of device the op runs. + /// + /// + /// + /// Given a list of device names, this operation returns the index of the device + /// this op runs. The length of the list is returned in two cases: + /// (1) Device does not exist in the given device list. + /// (2) It is in XLA compilation. + /// + /// + /// + /// + public static Tensor device_index(string[] device_names, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DeviceIndex", name) { args = new object[] { }, attrs = new Dictionary() { ["device_names"] = device_names } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return device_index_eager_fallback(device_names: device_names, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["device_names"] = device_names; + var _op = tf.OpDefLib._apply_op_helper("DeviceIndex", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "device_names", _op.get_attr("device_names") }; + _execute.record_gradient("DeviceIndex", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor device_index_eager_fallback(string[] device_names, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "device_names", device_names }; + var _result = _execute.execute("DeviceIndex", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("DeviceIndex", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// ~~%~~ This op is used as a placeholder in If branch functions. It doesn't provide a~~%~~ valid output when run, so must either be removed (e.g. replaced with a~~%~~ function input) or guaranteed not to be used (e.g. if mirroring an~~%~~ intermediate output needed for the gradient computation of the other branch).~~%~~ + /// + /// + /// The type of the output. + /// + /// + /// + /// The purported shape of the output. This is only used for shape inference; + /// the output will not necessarily have this shape. Can be a partial shape. + /// + /// + /// + public static Tensor fake_param(TF_DataType dtype, Shape shape, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeParam", name) { args = new object[] { }, attrs = new Dictionary() { ["dtype"] = dtype, ["shape"] = shape } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return fake_param_eager_fallback(dtype: dtype, shape: shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["dtype"] = dtype; + keywords["shape"] = shape; + var _op = tf.OpDefLib._apply_op_helper("FakeParam", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "shape", _op.get_attr("shape") }; + _execute.record_gradient("FakeParam", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor fake_param_eager_fallback(TF_DataType dtype, Shape shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "dtype", dtype, "shape", shape }; + var _result = _execute.execute("FakeParam", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("FakeParam", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Applies a for loop. + /// + /// + /// + /// ```python + /// output = input; + /// for i in range(start, limit, delta) + /// output = body(i, output); + /// ``` + /// + /// + /// + /// + /// + /// + /// + /// + /// A function that takes a list of tensors (int32, T) and returns another + /// list of tensors (T). + /// + /// + /// + public static Tensor[] _for(Tensor start, Tensor limit, Tensor delta, Tensors input, object body, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "For", name) { args = new object[] { start, limit, delta, input }, attrs = new Dictionary() { ["body"] = body } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return for_eager_fallback(start, limit, delta, input, body: body, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["start"] = start; + keywords["limit"] = limit; + keywords["delta"] = delta; + keywords["input"] = input; + keywords["body"] = body; + var _op = tf.OpDefLib._apply_op_helper("For", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op.get_attr("T"), "body", _op.get_attr("body") }; + _execute.record_gradient("For", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] for_eager_fallback(Tensor start, Tensor limit, Tensor delta, Tensor input, object body, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { start, limit, delta, input }; + object[] _attrs = new object[] { "body", body }; + var _result = _execute.execute("For", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("For", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// output = cond ? then_branch(input) : else_branch(input) + /// + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A function that takes 'inputs' and returns a list of tensors, whose + /// types are the same as what else_branch returns. + /// + /// + /// + /// + /// A function that takes 'inputs' and returns a list of tensors, whose + /// types are the same as what then_branch returns. + /// + /// + /// + /// + public static Tensor[] _if(Tensor cond, Tensors input, TF_DataType[] Tout, object then_branch, object else_branch, Shape[] output_shapes, string? name = null) { - public static Tensor[] partitioned_call(Tensors args, TF_DataType[] tout, EagerDefinedFunction f, - string config = "", string config_proto = "", string executor_type = "", string name = null) + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - var ctx = tf.Context; - if (ctx.executing_eagerly()) + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "If", name) { args = new object[] { cond, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["then_branch"] = then_branch, ["else_branch"] = else_branch, ["output_shapes"] = output_shapes } }); + return _fast_path_result; + } + catch (Exception) { - try - { - return tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "PartitionedCall", name, - args, tout, f, config, config_proto, executor_type)); - } - catch (Exception) - { + } + try + { + return if_eager_fallback(cond, input, Tout: Tout, then_branch: then_branch, else_branch: else_branch, output_shapes: output_shapes, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["cond"] = cond; + keywords["input"] = input; + keywords["Tout"] = Tout; + keywords["then_branch"] = then_branch; + keywords["else_branch"] = else_branch; + keywords["output_shapes"] = output_shapes; + var _op = tf.OpDefLib._apply_op_helper("If", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tcond", _op._get_attr_type("Tcond"), "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "then_branch", _op.get_attr("then_branch"), "else_branch", _op.get_attr("else_branch"), "output_shapes", _op.get_attr("output_shapes") }; + _execute.record_gradient("If", _op.inputs, _attrs, _result); + } + return _result; + } - } + public static Tensor[] if_eager_fallback(Tensor cond, Tensor input, TF_DataType[] Tout, object then_branch, object else_branch, Shape[] output_shapes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { cond, input }; + object[] _attrs = new object[] { "Tcond", cond.dtype, "then_branch", then_branch, "else_branch", else_branch, "output_shapes", output_shapes }; + var _result = _execute.execute("If", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("If", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// returns `f(inputs)`, where `f`'s body is placed and partitioned. + /// + /// + /// + /// Asynchronously executes a function, potentially across multiple devices but + /// within a single process. The kernel places and partitions a given function's + /// underlying graph, and executes each of the partitioned subgraphs as a function. + /// + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A function that takes 'args', a list of tensors, and returns 'output', + /// another list of tensors. Input and output types are specified by 'Tin' + /// and 'Tout'. The function body of f will be placed and partitioned across + /// devices, setting this op apart from the regular Call op. + /// + /// + /// + /// + /// + /// + public static Tensor[] partitioned_call(Tensors args, TF_DataType[] Tout, object f, string config = "", string config_proto = "", string executor_type = "", string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PartitionedCall", name) { args = new object[] { args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f, ["config"] = config, ["config_proto"] = config_proto, ["executor_type"] = executor_type } }); + return _fast_path_result; } + catch (Exception) + { + } + try + { + return partitioned_call_eager_fallback(args, Tout: Tout, f: f, config: config, config_proto: config_proto, executor_type: executor_type, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + if (config is null) + { + config = ""; + } + if (config_proto is null) + { + config_proto = ""; + } + if (executor_type is null) + { + executor_type = ""; + } + Dictionary keywords = new(); + keywords["args"] = args; + keywords["Tout"] = Tout; + keywords["f"] = f; + keywords["config"] = config; + keywords["config_proto"] = config_proto; + keywords["executor_type"] = executor_type; + var _op = tf.OpDefLib._apply_op_helper("PartitionedCall", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "f", _op.get_attr("f"), "config", _op.get_attr("config"), "config_proto", _op.get_attr("config_proto"), "executor_type", _op.get_attr("executor_type") }; + _execute.record_gradient("PartitionedCall", _op.inputs, _attrs, _result); + } + return _result; + } - if (config is null) + public static Tensor[] partitioned_call_eager_fallback(Tensor args, TF_DataType[] Tout, object f, string config, string config_proto, string executor_type, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { args }; + object[] _attrs = new object[] { "f", f, "config", config, "config_proto", config_proto, "executor_type", executor_type }; + var _result = _execute.execute("PartitionedCall", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("PartitionedCall", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// Runs function `f` on a remote device indicated by `target`. + /// + /// + /// + /// + /// + /// The type list for the return values. + /// + /// + /// + /// + /// The function to run remotely. + /// + /// + /// + public static Tensor[] remote_call(Tensor target, Tensors args, TF_DataType[] Tout, object f, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try { - config = ""; + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RemoteCall", name) { args = new object[] { target, args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f } }); + return _fast_path_result; } - if (config_proto is null) + catch (Exception) { - config_proto = ""; } - if (executor_type is null) + try { - executor_type = ""; + return remote_call_eager_fallback(target, args, Tout: Tout, f: f, name: name, ctx: _ctx); } - Dictionary kwargs = new(); - kwargs["args"] = args; - kwargs["Tout"] = tout; - kwargs["f"] = f; - kwargs["config"] = config; - kwargs["config_proto"] = config_proto; - kwargs["executor_type"] = executor_type; - var output = tf.OpDefLib._apply_op_helper("PartitionedCall", - name, kwargs); - var result = output.outputs; - if (_execute.must_record_gradient()) + catch (Exception) { - throw new NotImplementedException(); } - return result; } + Dictionary keywords = new(); + keywords["target"] = target; + keywords["args"] = args; + keywords["Tout"] = Tout; + keywords["f"] = f; + var _op = tf.OpDefLib._apply_op_helper("RemoteCall", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "f", _op.get_attr("f") }; + _execute.record_gradient("RemoteCall", _op.inputs, _attrs, _result); + } + return _result; + } - public static Tensor[] partitioned_call_eager_fallback(Tensors args, TF_DataType[] tout, EagerDefinedFunction f, - string config, string config_proto, string executor_type, string name, Context ctx) + public static Tensor[] remote_call_eager_fallback(Tensor target, Tensor args, TF_DataType[] Tout, object f, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { target, args }; + object[] _attrs = new object[] { "f", f }; + var _result = _execute.execute("RemoteCall", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("RemoteCall", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// returns `f(inputs)`, where `f`'s body is placed and partitioned. + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A function that takes 'args', a list of tensors, and returns 'output', + /// another list of tensors. Input and output types are specified by 'Tin' + /// and 'Tout'. The function body of f will be placed and partitioned across + /// devices, setting this op apart from the regular Call op. This op is + /// stateful. + /// + /// + /// + /// + /// + /// + public static Tensor[] stateful_partitioned_call(Tensors args, TF_DataType[] Tout, object f, string config = "", string config_proto = "", string executor_type = "", string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - // TODO(Rinne): implement it. - throw new NotImplementedException(); - if(config is null) + try { - config = ""; + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatefulPartitionedCall", name) { args = new object[] { args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f, ["config"] = config, ["config_proto"] = config_proto, ["executor_type"] = executor_type } }); + return _fast_path_result; } - if(config_proto is null) + catch (Exception) { - config_proto = ""; } - if(executor_type is null) + try { - executor_type = ""; + return stateful_partitioned_call_eager_fallback(args, Tout: Tout, f: f, config: config, config_proto: config_proto, executor_type: executor_type, name: name, ctx: _ctx); } - object[] attrs = new object[] + catch (Exception) { + } + } + if (config is null) + { + config = ""; + } + if (config_proto is null) + { + config_proto = ""; + } + if (executor_type is null) + { + executor_type = ""; + } + Dictionary keywords = new(); + keywords["args"] = args; + keywords["Tout"] = Tout; + keywords["f"] = f; + keywords["config"] = config; + keywords["config_proto"] = config_proto; + keywords["executor_type"] = executor_type; + var _op = tf.OpDefLib._apply_op_helper("StatefulPartitionedCall", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "f", _op.get_attr("f"), "config", _op.get_attr("config"), "config_proto", _op.get_attr("config_proto"), "executor_type", _op.get_attr("executor_type") }; + _execute.record_gradient("StatefulPartitionedCall", _op.inputs, _attrs, _result); + } + return _result; + } - }; + public static Tensor[] stateful_partitioned_call_eager_fallback(Tensor args, TF_DataType[] Tout, object f, string config, string config_proto, string executor_type, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { args }; + object[] _attrs = new object[] { "f", f, "config", config, "config_proto", config_proto, "executor_type", executor_type }; + var _result = _execute.execute("StatefulPartitionedCall", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("StatefulPartitionedCall", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// An n-way switch statement which calls a single branch function. + /// + /// + /// + /// An n-way switch statement, implementing the following: + /// ``` + /// switch (branch_index) { + /// case 0: + /// output = branches[0](input); + /// break; + /// case 1: + /// output = branches[1](input); + /// break; + /// ... + /// case [[nbranches-1]]: + /// default: + /// output = branches[nbranches-1](input); + /// break; + /// } + /// ``` + /// + /// This should only be used when the none of branches has stateful ops. + /// + /// + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A list of functions each of which takes 'inputs' and returns a list of + /// tensors, whose types are the same as what every other branch returns. + /// + /// + /// + /// + public static Tensor[] stateless_case(Tensor branch_index, Tensors input, TF_DataType[] Tout, object[] branches, Shape[] output_shapes, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessCase", name) { args = new object[] { branch_index, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["branches"] = branches, ["output_shapes"] = output_shapes } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return stateless_case_eager_fallback(branch_index, input, Tout: Tout, branches: branches, output_shapes: output_shapes, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["branch_index"] = branch_index; + keywords["input"] = input; + keywords["Tout"] = Tout; + keywords["branches"] = branches; + keywords["output_shapes"] = output_shapes; + var _op = tf.OpDefLib._apply_op_helper("StatelessCase", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "branches", _op.get_attr("branches"), "output_shapes", _op.get_attr("output_shapes") }; + _execute.record_gradient("StatelessCase", _op.inputs, _attrs, _result); } + return _result; + } - public static Tensor[] symbolic_gradient(Tensor[] input, TF_DataType[] Tout, NameAttrList f, string name = null) + public static Tensor[] stateless_case_eager_fallback(Tensor branch_index, Tensor input, TF_DataType[] Tout, object[] branches, Shape[] output_shapes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { branch_index, input }; + object[] _attrs = new object[] { "branches", branches, "output_shapes", output_shapes }; + var _result = _execute.execute("StatelessCase", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - var ctx = tf.Context; - if (ctx.executing_eagerly()) + _execute.record_gradient("StatelessCase", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// output = cond ? then_branch(input) : else_branch(input) + /// + /// + /// + /// + /// A list of output types. + /// + /// + /// + /// A function that takes 'inputs' and returns a list of tensors, whose + /// types are the same as what else_branch returns. + /// + /// + /// + /// + /// A function that takes 'inputs' and returns a list of tensors, whose + /// types are the same as what then_branch returns. + /// + /// + /// + /// + public static Tensor[] stateless_if(Tensor cond, Tensors input, TF_DataType[] Tout, object then_branch, object else_branch, Shape[] output_shapes, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessIf", name) { args = new object[] { cond, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["then_branch"] = then_branch, ["else_branch"] = else_branch, ["output_shapes"] = output_shapes } }); + return _fast_path_result; + } + catch (Exception) + { + } + try { - try - { - var _result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo( - tf.Context, "SymbolicGradient", name, input, Tout, f)); - return _result; - } - catch (Exception) - { + return stateless_if_eager_fallback(cond, input, Tout: Tout, then_branch: then_branch, else_branch: else_branch, output_shapes: output_shapes, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["cond"] = cond; + keywords["input"] = input; + keywords["Tout"] = Tout; + keywords["then_branch"] = then_branch; + keywords["else_branch"] = else_branch; + keywords["output_shapes"] = output_shapes; + var _op = tf.OpDefLib._apply_op_helper("StatelessIf", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tcond", _op._get_attr_type("Tcond"), "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "then_branch", _op.get_attr("then_branch"), "else_branch", _op.get_attr("else_branch"), "output_shapes", _op.get_attr("output_shapes") }; + _execute.record_gradient("StatelessIf", _op.inputs, _attrs, _result); + } + return _result; + } - } + public static Tensor[] stateless_if_eager_fallback(Tensor cond, Tensor input, TF_DataType[] Tout, object then_branch, object else_branch, Shape[] output_shapes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { cond, input }; + object[] _attrs = new object[] { "Tcond", cond.dtype, "then_branch", then_branch, "else_branch", else_branch, "output_shapes", output_shapes }; + var _result = _execute.execute("StatelessIf", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("StatelessIf", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// output = input; While (Cond(output)) { output = Body(output) } + /// + /// + /// + /// + /// A function takes 'input' and returns a tensor. If the tensor is + /// a scalar of non-boolean, the scalar is converted to a boolean + /// according to the following rule: if the scalar is a numerical + /// value, non-zero means True and zero means False; if the scalar is + /// a string, non-empty means True and empty means False. If the + /// tensor is not a scalar, non-emptiness means True and False + /// otherwise. + /// + /// This should only be used when the while condition and body functions + /// do not have stateful ops. + /// + /// + /// + /// + /// A function that takes a list of tensors and returns another + /// list of tensors. Both lists have the same types as specified + /// by T. + /// + /// + /// + /// + /// + public static Tensor[] stateless_while(Tensors input, object cond, object body, Shape[] output_shapes, int parallel_iterations = 10, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessWhile", name) { args = new object[] { input }, attrs = new Dictionary() { ["cond"] = cond, ["body"] = body, ["output_shapes"] = output_shapes, ["parallel_iterations"] = parallel_iterations } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return stateless_while_eager_fallback(input, cond: cond, body: body, output_shapes: output_shapes, parallel_iterations: parallel_iterations, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input"] = input; + keywords["cond"] = cond; + keywords["body"] = body; + keywords["output_shapes"] = output_shapes; + keywords["parallel_iterations"] = parallel_iterations; + var _op = tf.OpDefLib._apply_op_helper("StatelessWhile", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op.get_attr("T"), "cond", _op.get_attr("cond"), "body", _op.get_attr("body"), "output_shapes", _op.get_attr("output_shapes"), "parallel_iterations", _op._get_attr_int("parallel_iterations") }; + _execute.record_gradient("StatelessWhile", _op.inputs, _attrs, _result); + } + return _result; + } - try - { - return symbolic_gradient_eager_fallback(input, Tout, f, name, ctx); - } - catch (Exception) - { + public static Tensor[] stateless_while_eager_fallback(Tensor input, object cond, object body, Shape[] output_shapes, int parallel_iterations, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "cond", cond, "body", body, "output_shapes", output_shapes, "parallel_iterations", parallel_iterations }; + var _result = _execute.execute("StatelessWhile", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("StatelessWhile", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// Computes the gradient function for function f via backpropagation. + /// + /// + /// + /// + /// the type list for the input list. + /// + /// + /// + /// + /// The function we want to compute the gradient for. + /// + /// The function 'f' must be a numerical function which takes N inputs and + /// produces M outputs. Its gradient function 'g', which is computed by + /// this SymbolicGradient op is a function taking N + M inputs and + /// produces N outputs. + /// + /// I.e. if we have + /// (y1, y2, ..., y_M) = f(x1, x2, ..., x_N), + /// then, g is + /// (dL/dx1, dL/dx2, ..., dL/dx_N) = g(x1, x2, ..., x_N, + /// dL/dy1, dL/dy2, ..., dL/dy_M), + /// + /// where L is a scalar-value function of (x1, x2, ..., xN) (e.g., the + /// loss function). dL/dx_i is the partial derivative of L with respect + /// to x_i. + /// + /// (Needs some math expert to say the comment above better.) + /// + /// + /// + public static Tensor[] symbolic_gradient(Tensors input, TF_DataType[] Tout, object f, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SymbolicGradient", name) { args = new object[] { input }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return symbolic_gradient_eager_fallback(input, Tout: Tout, f: f, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input"] = input; + keywords["Tout"] = Tout; + keywords["f"] = f; + var _op = tf.OpDefLib._apply_op_helper("SymbolicGradient", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "Tin", _op.get_attr("Tin"), "Tout", _op.get_attr("Tout"), "f", _op.get_attr("f") }; + _execute.record_gradient("SymbolicGradient", _op.inputs, _attrs, _result); + } + return _result; + } - } + public static Tensor[] symbolic_gradient_eager_fallback(Tensor input, TF_DataType[] Tout, object f, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "f", f }; + var _result = _execute.execute("SymbolicGradient", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("SymbolicGradient", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// Converts a tensor to a scalar predicate. + /// + /// + /// + /// Converts a tensor to a scalar predicate with the following rules: + /// + /// - For 0D tensors, truthiness is determined by comparing against a "zero" + /// value. For numerical types it is the obvious zero. For strings it is the + /// empty string. + /// + /// - For >0D tensors, truthiness is determined by looking at the number of + /// elements. If has zero elements, then the result is false. Otherwise the + /// result is true. + /// + /// This matches the behavior of If and While for determining if a tensor counts + /// as true/false for a branch condition. + /// + /// + /// + /// + public static Tensor to_bool(Tensor input, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ToBool", name) { args = new object[] { input }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { } - var op = tf.OpDefLib._apply_op_helper("SymbolicGradient", name, new object[] { input, Tout, f }); - var result = op.outputs; - if (_execute.must_record_gradient()) + try { - throw new NotImplementedException(); + return to_bool_eager_fallback(input, name: name, ctx: _ctx); } - return result; + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input"] = input; + var _op = tf.OpDefLib._apply_op_helper("ToBool", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("ToBool", _op.inputs, _attrs, _result); } + return _result[0]; + } - public static Tensor[] symbolic_gradient_eager_fallback(Tensor[] input, TF_DataType[] Tout, NameAttrList f, string name, Context ctx) + public static Tensor to_bool_eager_fallback(Tensor input, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "T", input.dtype }; + var _result = _execute.execute("ToBool", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ToBool", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// output = input; While (Cond(output)) { output = Body(output) } + /// + /// + /// + /// + /// A function takes 'input' and returns a tensor. If the tensor is + /// a scalar of non-boolean, the scalar is converted to a boolean + /// according to the following rule: if the scalar is a numerical + /// value, non-zero means True and zero means False; if the scalar is + /// a string, non-empty means True and empty means False. If the + /// tensor is not a scalar, non-emptiness means True and False + /// otherwise. + /// + /// + /// + /// + /// A function that takes a list of tensors and returns another + /// list of tensors. Both lists have the same types as specified + /// by T. + /// + /// + /// + /// + /// + public static Tensor[] _while(Tensors input, object cond, object body, Shape[] output_shapes, int parallel_iterations = 10, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - object[] attrs = new object[] { "Tin", input, "Tout", Tout, "f", f }; - var result = _execute.execute("SymbolicGradient", Tout.Length, input, attrs, ctx, name); - if (_execute.must_record_gradient()) + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "While", name) { args = new object[] { input }, attrs = new Dictionary() { ["cond"] = cond, ["body"] = body, ["output_shapes"] = output_shapes, ["parallel_iterations"] = parallel_iterations } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return while_eager_fallback(input, cond: cond, body: body, output_shapes: output_shapes, parallel_iterations: parallel_iterations, name: name, ctx: _ctx); + } + catch (Exception) { - throw new NotImplementedException(); } - return result; } + Dictionary keywords = new(); + keywords["input"] = input; + keywords["cond"] = cond; + keywords["body"] = body; + keywords["output_shapes"] = output_shapes; + keywords["parallel_iterations"] = parallel_iterations; + var _op = tf.OpDefLib._apply_op_helper("While", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op.get_attr("T"), "cond", _op.get_attr("cond"), "body", _op.get_attr("body"), "output_shapes", _op.get_attr("output_shapes"), "parallel_iterations", _op._get_attr_int("parallel_iterations") }; + _execute.record_gradient("While", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] while_eager_fallback(Tensor input, object cond, object body, Shape[] output_shapes, int parallel_iterations, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "cond", cond, "body", body, "output_shapes", output_shapes, "parallel_iterations", parallel_iterations }; + var _result = _execute.execute("While", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("While", _inputs_flat, _attrs, _result); + } + return _result; } } diff --git a/src/TensorFlowNET.Core/Operations/gen_list_ops.cs b/src/TensorFlowNET.Core/Operations/gen_list_ops.cs new file mode 100644 index 00000000..e7253986 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/gen_list_ops.cs @@ -0,0 +1,1227 @@ +/*Wrappers around TensorFlow ops. This file is MACHINE GENERATED! Do not edit.*/ + +using Tensorflow.Eager; +using Tensorflow.Contexts; +using static Tensorflow.Binding; + +namespace Tensorflow; + +public static class gen_list_ops +{ + /// + /// Creates and returns an empty tensor list. + /// + /// + /// + /// All list elements must be tensors of dtype element_dtype and shape compatible + /// with element_shape. + /// + /// handle: an empty tensor list. + /// element_dtype: the type of elements in the list. + /// element_shape: a shape compatible with that of elements in the list. + /// + /// + /// + /// + /// + /// + public static Tensor empty_tensor_list(Tensor element_shape, Tensor max_num_elements, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EmptyTensorList", name) { args = new object[] { element_shape, max_num_elements }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return empty_tensor_list_eager_fallback(element_shape, max_num_elements, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["element_shape"] = element_shape; + keywords["max_num_elements"] = max_num_elements; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("EmptyTensorList", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("EmptyTensorList", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor empty_tensor_list_eager_fallback(Tensor element_shape, Tensor max_num_elements, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { element_shape, max_num_elements }; + object[] _attrs = new object[] { "element_dtype", element_dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("EmptyTensorList", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("EmptyTensorList", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Concats all tensors in the list along the 0th dimension. + /// + /// + /// + /// Requires that all tensors have the same shape except the first dimension. + /// + /// input_handle: The input list. + /// tensor: The concated result. + /// lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient. + /// + /// + /// + /// + /// + /// + /// + public static Tensor[] tensor_list_concat(Tensor input_handle, TF_DataType element_dtype, Shape element_shape = null, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcat", name) { args = new object[] { input_handle }, attrs = new Dictionary() { ["element_dtype"] = element_dtype, ["element_shape"] = element_shape } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return tensor_list_concat_eager_fallback(input_handle, element_dtype: element_dtype, element_shape: element_shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["element_dtype"] = element_dtype; + keywords["element_shape"] = element_shape; + var _op = tf.OpDefLib._apply_op_helper("TensorListConcat", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "element_shape", _op.get_attr("element_shape") }; + _execute.record_gradient("TensorListConcat", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] tensor_list_concat_eager_fallback(Tensor input_handle, TF_DataType element_dtype, Shape element_shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle }; + object[] _attrs = new object[] { "element_dtype", element_dtype, "element_shape", element_shape }; + var _result = _execute.execute("TensorListConcat", 2, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListConcat", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_concat_lists(Tensor input_a, Tensor input_b, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcatLists", name) { args = new object[] { input_a, input_b }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_concat_lists_eager_fallback(input_a, input_b, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_a"] = input_a; + keywords["input_b"] = input_b; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListConcatLists", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListConcatLists", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_concat_lists_eager_fallback(Tensor input_a, Tensor input_b, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_a, input_b }; + object[] _attrs = new object[] { "element_dtype", element_dtype }; + var _result = _execute.execute("TensorListConcatLists", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListConcatLists", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Concats all tensors in the list along the 0th dimension. + /// + /// + /// + /// Requires that all tensors have the same shape except the first dimension. + /// + /// input_handle: The input list. + /// element_shape: The shape of the uninitialized elements in the list. If the first + /// dimension is not -1, it is assumed that all list elements have the same + /// leading dim. + /// leading_dims: The list of leading dims of uninitialized list elements. Used if + /// the leading dim of input_handle.element_shape or the element_shape input arg + /// is not already set. + /// tensor: The concated result. + /// lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient. + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor[] tensor_list_concat_v2(Tensor input_handle, Tensor element_shape, Tensor leading_dims, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcatV2", name) { args = new object[] { input_handle, element_shape, leading_dims }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return tensor_list_concat_v2_eager_fallback(input_handle, element_shape, leading_dims, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["element_shape"] = element_shape; + keywords["leading_dims"] = leading_dims; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListConcatV2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListConcatV2", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] tensor_list_concat_v2_eager_fallback(Tensor input_handle, Tensor element_shape, Tensor leading_dims, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, element_shape, leading_dims }; + object[] _attrs = new object[] { "element_dtype", element_dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListConcatV2", 2, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListConcatV2", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// The shape of the elements of the given list, as a tensor. + /// + /// + /// + /// input_handle: the list + /// element_shape: the shape of elements of the list + /// + /// + /// + /// + /// + public static Tensor tensor_list_element_shape(Tensor input_handle, TF_DataType shape_type, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListElementShape", name) { args = new object[] { input_handle }, attrs = new Dictionary() { ["shape_type"] = shape_type } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_element_shape_eager_fallback(input_handle, shape_type: shape_type, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["shape_type"] = shape_type; + var _op = tf.OpDefLib._apply_op_helper("TensorListElementShape", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListElementShape", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_element_shape_eager_fallback(Tensor input_handle, TF_DataType shape_type, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle }; + object[] _attrs = new object[] { "shape_type", shape_type }; + var _result = _execute.execute("TensorListElementShape", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListElementShape", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a TensorList which, when stacked, has the value of `tensor`. + /// + /// + /// + /// Each tensor in the result list corresponds to one row of the input tensor. + /// + /// tensor: The input tensor. + /// output_handle: The list. + /// + /// + /// + /// + /// + public static Tensor tensor_list_from_tensor(Tensor tensor, Tensor element_shape, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListFromTensor", name) { args = new object[] { tensor, element_shape }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_from_tensor_eager_fallback(tensor, element_shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["tensor"] = tensor; + keywords["element_shape"] = element_shape; + var _op = tf.OpDefLib._apply_op_helper("TensorListFromTensor", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListFromTensor", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_from_tensor_eager_fallback(Tensor tensor, Tensor element_shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { tensor, element_shape }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListFromTensor", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListFromTensor", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a Tensor by indexing into the TensorList. + /// + /// + /// + /// Each row in the produced Tensor corresponds to the element in the TensorList + /// specified by the given index (see `tf.gather`). + /// + /// input_handle: The input tensor list. + /// indices: The indices used to index into the list. + /// values: The tensor. + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_gather(Tensor input_handle, Tensor indices, Tensor element_shape, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListGather", name) { args = new object[] { input_handle, indices, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_gather_eager_fallback(input_handle, indices, element_shape, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["indices"] = indices; + keywords["element_shape"] = element_shape; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListGather", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListGather", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_gather_eager_fallback(Tensor input_handle, Tensor indices, Tensor element_shape, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, indices, element_shape }; + object[] _attrs = new object[] { "element_dtype", element_dtype }; + var _result = _execute.execute("TensorListGather", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListGather", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_get_item(Tensor input_handle, Tensor index, Tensor element_shape, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListGetItem", name) { args = new object[] { input_handle, index, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_get_item_eager_fallback(input_handle, index, element_shape, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["index"] = index; + keywords["element_shape"] = element_shape; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListGetItem", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListGetItem", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_get_item_eager_fallback(Tensor input_handle, Tensor index, Tensor element_shape, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, index, element_shape }; + object[] _attrs = new object[] { "element_dtype", element_dtype }; + var _result = _execute.execute("TensorListGetItem", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListGetItem", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Returns the number of tensors in the input tensor list. + /// + /// + /// + /// input_handle: the input list + /// length: the number of tensors in the list + /// + /// + /// + /// + public static Tensor tensor_list_length(Tensor input_handle, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListLength", name) { args = new object[] { input_handle }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_length_eager_fallback(input_handle, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + var _op = tf.OpDefLib._apply_op_helper("TensorListLength", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("TensorListLength", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_length_eager_fallback(Tensor input_handle, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("TensorListLength", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListLength", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Returns the last element of the input list as well as a list with all but that element. + /// + /// + /// + /// Fails if the list is empty. + /// + /// input_handle: the input list + /// tensor: the withdrawn last element of the list + /// element_dtype: the type of elements in the list + /// element_shape: the shape of the output tensor + /// + /// + /// + /// + /// + /// + public static Tensor[] tensor_list_pop_back(Tensor input_handle, Tensor element_shape, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPopBack", name) { args = new object[] { input_handle, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result; + } + catch (Exception) + { + } + try + { + return tensor_list_pop_back_eager_fallback(input_handle, element_shape, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["element_shape"] = element_shape; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListPopBack", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListPopBack", _op.inputs, _attrs, _result); + } + return _result; + } + + public static Tensor[] tensor_list_pop_back_eager_fallback(Tensor input_handle, Tensor element_shape, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, element_shape }; + object[] _attrs = new object[] { "element_dtype", element_dtype }; + var _result = _execute.execute("TensorListPopBack", 2, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListPopBack", _inputs_flat, _attrs, _result); + } + return _result; + } + /// + /// Returns a list which has the passed-in `Tensor` as last element and the other elements of the given list in `input_handle`. + /// + /// + /// + /// tensor: The tensor to put on the list. + /// input_handle: The old list. + /// output_handle: A list with the elements of the old list followed by tensor. + /// element_dtype: the type of elements in the list. + /// element_shape: a shape compatible with that of elements in the list. + /// + /// + /// + /// + /// + public static Tensor tensor_list_push_back(Tensor input_handle, Tensor tensor, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPushBack", name) { args = new object[] { input_handle, tensor }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_push_back_eager_fallback(input_handle, tensor, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["tensor"] = tensor; + var _op = tf.OpDefLib._apply_op_helper("TensorListPushBack", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListPushBack", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_push_back_eager_fallback(Tensor input_handle, Tensor tensor, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, tensor }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype }; + var _result = _execute.execute("TensorListPushBack", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListPushBack", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_push_back_batch(Tensor input_handles, Tensor tensor, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPushBackBatch", name) { args = new object[] { input_handles, tensor }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_push_back_batch_eager_fallback(input_handles, tensor, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handles"] = input_handles; + keywords["tensor"] = tensor; + var _op = tf.OpDefLib._apply_op_helper("TensorListPushBackBatch", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListPushBackBatch", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_push_back_batch_eager_fallback(Tensor input_handles, Tensor tensor, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handles, tensor }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype }; + var _result = _execute.execute("TensorListPushBackBatch", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListPushBackBatch", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// List of the given size with empty elements. + /// + /// + /// + /// element_shape: the shape of the future elements of the list + /// num_elements: the number of elements to reserve + /// handle: the output list + /// element_dtype: the desired type of elements in the list. + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_reserve(Tensor element_shape, Tensor num_elements, TF_DataType element_dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListReserve", name) { args = new object[] { element_shape, num_elements }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_reserve_eager_fallback(element_shape, num_elements, element_dtype: element_dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["element_shape"] = element_shape; + keywords["num_elements"] = num_elements; + keywords["element_dtype"] = element_dtype; + var _op = tf.OpDefLib._apply_op_helper("TensorListReserve", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListReserve", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_reserve_eager_fallback(Tensor element_shape, Tensor num_elements, TF_DataType element_dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { element_shape, num_elements }; + object[] _attrs = new object[] { "element_dtype", element_dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListReserve", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListReserve", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Resizes the list. + /// + /// + /// + /// + /// input_handle: the input list + /// size: size of the output list + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_resize(Tensor input_handle, Tensor size, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListResize", name) { args = new object[] { input_handle, size }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_resize_eager_fallback(input_handle, size, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["size"] = size; + var _op = tf.OpDefLib._apply_op_helper("TensorListResize", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("TensorListResize", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_resize_eager_fallback(Tensor input_handle, Tensor size, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, size }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("TensorListResize", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListResize", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a TensorList by indexing into a Tensor. + /// + /// + /// + /// Each member of the TensorList corresponds to one row of the input tensor, + /// specified by the given index (see `tf.gather`). + /// + /// tensor: The input tensor. + /// indices: The indices used to index into the list. + /// element_shape: The shape of the elements in the list (can be less specified than + /// the shape of the tensor). + /// output_handle: The TensorList. + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_scatter(Tensor tensor, Tensor indices, Tensor element_shape, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatter", name) { args = new object[] { tensor, indices, element_shape }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_scatter_eager_fallback(tensor, indices, element_shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["tensor"] = tensor; + keywords["indices"] = indices; + keywords["element_shape"] = element_shape; + var _op = tf.OpDefLib._apply_op_helper("TensorListScatter", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListScatter", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_scatter_eager_fallback(Tensor tensor, Tensor indices, Tensor element_shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { tensor, indices, element_shape }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListScatter", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListScatter", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Scatters tensor at indices in an input list. + /// + /// + /// + /// Each member of the TensorList corresponds to one row of the input tensor, + /// specified by the given index (see `tf.gather`). + /// + /// input_handle: The list to scatter into. + /// tensor: The input tensor. + /// indices: The indices used to index into the list. + /// output_handle: The TensorList. + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_scatter_into_existing_list(Tensor input_handle, Tensor tensor, Tensor indices, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatterIntoExistingList", name) { args = new object[] { input_handle, tensor, indices }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_scatter_into_existing_list_eager_fallback(input_handle, tensor, indices, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["tensor"] = tensor; + keywords["indices"] = indices; + var _op = tf.OpDefLib._apply_op_helper("TensorListScatterIntoExistingList", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListScatterIntoExistingList", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_scatter_into_existing_list_eager_fallback(Tensor input_handle, Tensor tensor, Tensor indices, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, tensor, indices }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype }; + var _result = _execute.execute("TensorListScatterIntoExistingList", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListScatterIntoExistingList", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a TensorList by indexing into a Tensor. + /// + /// + /// + /// Each member of the TensorList corresponds to one row of the input tensor, + /// specified by the given index (see `tf.gather`). + /// + /// tensor: The input tensor. + /// indices: The indices used to index into the list. + /// element_shape: The shape of the elements in the list (can be less specified than + /// the shape of the tensor). + /// num_elements: The size of the output list. Must be large enough to accommodate + /// the largest index in indices. If -1, the list is just large enough to include + /// the largest index in indices. + /// output_handle: The TensorList. + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_scatter_v2(Tensor tensor, Tensor indices, Tensor element_shape, Tensor num_elements, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatterV2", name) { args = new object[] { tensor, indices, element_shape, num_elements }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_scatter_v2_eager_fallback(tensor, indices, element_shape, num_elements, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["tensor"] = tensor; + keywords["indices"] = indices; + keywords["element_shape"] = element_shape; + keywords["num_elements"] = num_elements; + var _op = tf.OpDefLib._apply_op_helper("TensorListScatterV2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListScatterV2", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_scatter_v2_eager_fallback(Tensor tensor, Tensor indices, Tensor element_shape, Tensor num_elements, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { tensor, indices, element_shape, num_elements }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListScatterV2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListScatterV2", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_set_item(Tensor input_handle, Tensor index, Tensor item, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListSetItem", name) { args = new object[] { input_handle, index, item }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_set_item_eager_fallback(input_handle, index, item, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["index"] = index; + keywords["item"] = item; + var _op = tf.OpDefLib._apply_op_helper("TensorListSetItem", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype") }; + _execute.record_gradient("TensorListSetItem", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_set_item_eager_fallback(Tensor input_handle, Tensor index, Tensor item, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, index, item }; + object[] _attrs = new object[] { "element_dtype", item.dtype }; + var _result = _execute.execute("TensorListSetItem", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListSetItem", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Splits a tensor into a list. + /// + /// + /// + /// list[i] corresponds to lengths[i] tensors from the input tensor. + /// The tensor must have rank at least 1 and contain exactly sum(lengths) elements. + /// + /// tensor: The input tensor. + /// element_shape: A shape compatible with that of elements in the tensor. + /// lengths: Vector of sizes of the 0th dimension of tensors in the list. + /// output_handle: The list. + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_split(Tensor tensor, Tensor element_shape, Tensor lengths, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListSplit", name) { args = new object[] { tensor, element_shape, lengths }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_split_eager_fallback(tensor, element_shape, lengths, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["tensor"] = tensor; + keywords["element_shape"] = element_shape; + keywords["lengths"] = lengths; + var _op = tf.OpDefLib._apply_op_helper("TensorListSplit", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "shape_type", _op._get_attr_type("shape_type") }; + _execute.record_gradient("TensorListSplit", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_split_eager_fallback(Tensor tensor, Tensor element_shape, Tensor lengths, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { tensor, element_shape, lengths }; + object[] _attrs = new object[] { "element_dtype", tensor.dtype, "shape_type", element_shape.dtype }; + var _result = _execute.execute("TensorListSplit", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListSplit", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Stacks all tensors in the list. + /// + /// + /// + /// Requires that all tensors have the same shape. + /// + /// input_handle: the input list + /// tensor: the gathered result + /// num_elements: optional. If not -1, the number of elements in the list. + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor tensor_list_stack(Tensor input_handle, Tensor element_shape, TF_DataType element_dtype, int num_elements = -1, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListStack", name) { args = new object[] { input_handle, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype, ["num_elements"] = num_elements } }); + return _fast_path_result[0]; + } + catch (Exception) + { + } + try + { + return tensor_list_stack_eager_fallback(input_handle, element_shape, element_dtype: element_dtype, num_elements: num_elements, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input_handle"] = input_handle; + keywords["element_shape"] = element_shape; + keywords["element_dtype"] = element_dtype; + keywords["num_elements"] = num_elements; + var _op = tf.OpDefLib._apply_op_helper("TensorListStack", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "element_dtype", _op._get_attr_type("element_dtype"), "num_elements", _op._get_attr_int("num_elements") }; + _execute.record_gradient("TensorListStack", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor tensor_list_stack_eager_fallback(Tensor input_handle, Tensor element_shape, TF_DataType element_dtype, int num_elements, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input_handle, element_shape }; + object[] _attrs = new object[] { "element_dtype", element_dtype, "num_elements", num_elements }; + var _result = _execute.execute("TensorListStack", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("TensorListStack", _inputs_flat, _attrs, _result); + } + return _result[0]; + } +} diff --git a/src/TensorFlowNET.Core/Operations/list_ops.cs b/src/TensorFlowNET.Core/Operations/list_ops.cs new file mode 100644 index 00000000..c5e83ee4 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/list_ops.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Eager; + +namespace Tensorflow.Operations +{ + internal class list_ops + { + private static void _set_handle_data(Tensor list_handle, Shape element_shape, TF_DataType element_dtype) + { + if(list_handle is EagerTensor eagerTensor) + { + var handle_data = new CppShapeInferenceResult.Types.HandleData(); + handle_data.IsSet = true; + handle_data.ShapeAndType.Add(new CppShapeInferenceResult.Types.HandleShapeAndType() + { + Shape = element_shape.as_proto(), + Dtype = element_dtype.as_datatype_enum(), + Type = new FullTypeDef() { TypeId = FullTypeId.TftArray } + }); + list_handle.HandleData = handle_data; + } + } + + private static Tensor _build_element_shape(Shape? shape) + { + if(shape is null || shape.IsNull) + { + return ops.convert_to_tensor(-1); + } + else + { + return ops.convert_to_tensor(shape); + } + } + + public static Tensor tensor_list_reserve(Shape? shape, Tensor num_elements, TF_DataType element_dtype, string name = null) + { + var result = gen_list_ops.tensor_list_reserve(_build_element_shape(shape), num_elements, element_dtype, name); + _set_handle_data(result, shape, element_dtype); + return result; + } + + public static Tensor tensor_list_from_tensor(Tensor tensor, Shape element_shape, string? name = null) + { + var result = gen_list_ops.tensor_list_from_tensor(tensor, _build_element_shape(element_shape), name); + _set_handle_data(result, tensor.shape, tensor.dtype); + return result; + } + + public static Tensor tensor_list_get_item(Tensor input_handle, Tensor index, TF_DataType element_dtype, + Shape? element_shape = null, string? name = null) + { + return gen_list_ops.tensor_list_get_item(input_handle, index, _build_element_shape(element_shape), + element_dtype, name); + } + + public static Tensor tensor_list_set_item(Tensor input_handle, Tensor index, Tensor item, + bool resize_if_index_out_of_bounds = false, string? name = null) + { + if (resize_if_index_out_of_bounds) + { + var input_list_size = gen_list_ops.tensor_list_length(input_handle); + input_handle = control_flow_ops.cond(index >= input_list_size, + () => gen_list_ops.tensor_list_resize(input_handle, index + 1), + () => input_handle); + } + var output_handle = gen_list_ops.tensor_list_set_item(input_handle, index, item, name); + handle_data_util.copy_handle_data(input_handle, output_handle); + return output_handle; + } + + public static Tensor tensor_list_stack(Tensor input_handle, TF_DataType element_dtype, int num_elements = -1, + Shape? element_shape = null, string? name = null) + { + return gen_list_ops.tensor_list_stack(input_handle, _build_element_shape(element_shape), element_dtype, num_elements, name); + } + + public static Tensor tensor_list_gather(Tensor input_handle, Tensor indices, TF_DataType element_dtype, + Shape? element_shape = null, string? name = null) + { + return gen_list_ops.tensor_list_gather(input_handle, indices, _build_element_shape(element_shape), element_dtype, name); + } + + public static Tensor tensor_list_scatter(Tensor tensor, Tensor indices, Shape? element_shape = null, Tensor? input_handle = null, + string? name = null) + { + if(input_handle is not null) + { + var output_handle = gen_list_ops.tensor_list_scatter_into_existing_list(input_handle, tensor, indices, name); + handle_data_util.copy_handle_data(input_handle, output_handle); + return output_handle; + } + else + { + var output_handle = gen_list_ops.tensor_list_scatter_v2(tensor, indices, _build_element_shape(element_shape), + constant_op.constant(-1), name); + _set_handle_data(output_handle, element_shape, tensor.dtype); + return output_handle; + } + } + + public static Tensor empty_tensor_list(Shape? element_shape, TF_DataType element_dtype, int max_num_elements = -1, + string? name = null) + { + return gen_list_ops.empty_tensor_list(_build_element_shape(element_shape), element_dtype: element_dtype, + max_num_elements: ops.convert_to_tensor(max_num_elements, dtype: dtypes.int32), name: name); + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/tensor_array_ops.cs b/src/TensorFlowNET.Core/Operations/tensor_array_ops.cs index 7d2da544..6be0706c 100644 --- a/src/TensorFlowNET.Core/Operations/tensor_array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/tensor_array_ops.cs @@ -13,11 +13,23 @@ namespace Tensorflow /// public static TensorArray build_ta_with_new_flow(TensorArray old_ta, Tensor flow) { - var new_ta = tf.TensorArray( - dtype: old_ta.dtype, - infer_shape: old_ta.infer_shape, + if (!tf.Context.executing_eagerly() && old_ta is not _GraphTensorArrayV2 && control_flow_util.EnableControlFlowV2(ops.get_default_graph())) + { + throw new NotImplementedException("Attempting to build a graph-mode TF2-style " + + "TensorArray from either an eager-mode " + + "TensorArray or a TF1-style TensorArray. " + + "This is not currently supported. You may be " + + "attempting to capture a TensorArray " + + "inside a tf.function or tf.data map function. " + + "Instead, construct a new TensorArray inside " + + "the function."); + } + var new_ta = TensorArray.Create(old_ta.dtype, handle: old_ta.handle, flow: flow, infer_shape: old_ta.infer_shape, colocate_with_first_write_call: old_ta.colocate_with_first_write_call); - + new_ta._dynamic_size = old_ta._dynamic_size; + new_ta._size = old_ta._size; + new_ta._colocate_with = old_ta._colocate_with; + new_ta._element_shape = old_ta._element_shape; return new_ta; } diff --git a/src/TensorFlowNET.Core/Operations/while_v2.cs b/src/TensorFlowNET.Core/Operations/while_v2.cs new file mode 100644 index 00000000..7ee3e9e8 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/while_v2.cs @@ -0,0 +1,401 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Tensorflow.Common.Extensions; +using Tensorflow.Common.Types; +using Tensorflow.Eager; +using Tensorflow.Framework; +using Tensorflow.Framework.Models; +using Tensorflow.Graphs; +using static Tensorflow.Binding; + +namespace Tensorflow.Operations +{ + class _OperationWithOutputs : Operation + { + public _OperationWithOutputs(IntPtr handle, Graph g = null) + { + _handle = handle; + _graph = g; + _outputs = null; + g._add_op(this); + } + } + internal class while_v2 + { + public static Tensor[] while_loop(Func cond, + Func body, + Tensors loop_vars, + int maximum_iterations = -1, + int parallel_iterations = 10, + string name = null, + bool back_prop = true, + bool return_same_structure = true) + { + var orig_loop_vars = loop_vars; + var flat_orig_loop_vars = orig_loop_vars.Flatten().ToArray(); + int len_orig_loop_vars = orig_loop_vars.Length; + + loop_vars = _tensor_array_to_flow(loop_vars); + loop_vars = Nest.MapStructure(x => _convert_to_tensor_or_indexed_slices(x, TF_DataType.DtInvalid, null), loop_vars).ToTensors(); + + var loop_vars_signature = Nest.MapStructure(x => new TensorSpec(x.shape, x.dtype), _tensor_array_to_flow(loop_vars)); + + var flat_shape_invariants = Nest.Flatten(loop_vars_signature).Select(x => x.shape).ToArray(); + + if(string.IsNullOrEmpty(name)) + { + name = "while"; + } + + return tf_with(ops.name_scope(name), nameScopeWhile => + { + string scope = (nameScopeWhile as ops.NameScope).scope_name; + string cond_name = control_flow_util.unique_fn_name(scope, "cond"); + string body_name = control_flow_util.unique_fn_name(scope, "body"); + + var maximum_iterations_loop_var = _build_maximum_iterations_loop_var(maximum_iterations); + var loop_counter = constant_op.constant(0, maximum_iterations == -1 ? TF_DataType.DtInvalid : maximum_iterations_loop_var.dtype, + name: "loop_counter"); + loop_vars = new Tensor[] { loop_counter, maximum_iterations_loop_var }.Concat(loop_vars).ToArray(); + + var func_graph_signature = new TensorSpec[] {TensorSpec.FromTensor(loop_counter),TensorSpec.FromTensor(maximum_iterations_loop_var)} + .Concat(loop_vars_signature.Flatten()).ToArray(); + + // TODO(Rinne): possible wrong implemenation here. + var add_control_dependencies = false; + + object[] wrapped_cond(object[] inputs) + { + Tensor loop_counter = (Tensor)inputs[0]; + Tensor maximum_iterations_arg = (Tensor)inputs[1]; + Tensor[] args = inputs.Skip(2).Select(x => (Tensor)x).ToArray(); + var pred = cond(_pack_sequence_as(loop_vars_signature, flat_orig_loop_vars, args)); + if(pred.shape.IsNull || pred.shape.ndim > 0) + { + pred = array_ops.squeeze(pred); + } + if(maximum_iterations == -1) + { + return new object[] { pred }; + } + else + { + return new object[] { math_ops.logical_and(loop_counter < maximum_iterations_arg, pred) }; + } + } + + var cond_graph = FuncGraph.func_graph_from_func("cond", wrapped_cond, null, + null, signature: func_graph_signature, add_control_dependencies: add_control_dependencies); + + bool stateful_parallelism = false; + + object[] wrapped_body(object[] inputs) + { + Tensor loop_counter = (Tensor)inputs[0]; + Tensor maximum_iterations_arg = (Tensor)inputs[1]; + Tensor[] args = inputs.Skip(2).Select(x => (Tensor)x).ToArray(); + + _copy_handle_data(loop_vars.Flatten().Skip(2), args); + + foreach(var t in cond_graph.external_captures) + { + var graph = (FuncGraph)(ops.get_default_graph()); + graph.capture(t); + } + + var outputs = body(_pack_sequence_as(loop_vars_signature, flat_orig_loop_vars, args)); + outputs = _tensor_array_to_flow(outputs); + + return new object[] { loop_counter + 1, maximum_iterations_arg }.Concat(outputs).ToArray(); + } + + var body_graph = FuncGraph.func_graph_from_func("body", wrapped_body, null, null, func_graph_signature, + add_control_dependencies: add_control_dependencies, acd_record_initial_resource_uses: stateful_parallelism); + + // TODO(Rinne): possible wrong implementation here. + NestList loop_vars_list = new(new Tensors[] { loop_vars, body_graph.external_captures.ToTensors() }); + body_graph.Outputs.AddRange(body_graph.internal_captures); + + cond_graph.as_default(); + int num_cond_captures = cond_graph.external_captures.Length; + Debug.Assert(cond_graph.external_captures.SequenceEqual(body_graph.external_captures.Take(num_cond_captures).ToArray())); + _duplicate_body_captures_in_cond(cond_graph, body_graph.external_captures.Skip(num_cond_captures).ToArray()); + cond_graph.Exit(); + + int first_loop_var_index = 2; + + int num_flattened_oututs = orig_loop_vars.Length; + int num_original_outputs = body_graph.Outputs.Length; + if (back_prop && control_flow_util.output_all_intermediates()) + { + var intermediate_tensors = _get_intermediates(body_graph); + + foreach(var intermediate_tensor in intermediate_tensors) + { + var tensor_list = list_ops.empty_tensor_list(intermediate_tensor.shape, intermediate_tensor.dtype, maximum_iterations); + loop_vars_list.Values.Add(tensor_list); + + cond_graph.as_default(); + cond_graph.capture(tensor_list); + cond_graph.Exit(); + + body_graph.as_default(); + var appended_tensor_list = gen_ops.tensor_list_push_back(tensor_list, intermediate_tensor); + body_graph.Outputs.Add(appended_tensor_list); + body_graph.Exit(); + } + } + + List flattened_loop_vars = new(); + foreach(var item in loop_vars_list.Values) + { + flattened_loop_vars.AddRange(item.Flatten()); + } + // skip the check + + // TODO(Rinne): deal with control dependencies + var output_shapes = body_graph.Outputs.Select(t => t.shape).ToArray(); + var span = new Span(output_shapes).Slice(first_loop_var_index, num_flattened_oututs); + for(int i = 0; i < span.Length; i++) + { + span[i] = flat_shape_invariants[i]; + } + + Tensor[] outputs = _build_while_op(flattened_loop_vars.ToArray(), cond_graph, body_graph, output_shapes, parallel_iterations, + (nameScopeWhile as ops.NameScope).scope_name, num_original_outputs, stateful_parallelism); + + if (!ops.get_default_graph().building_function) + { + outputs = outputs.Select(t => array_ops.identity(t)).ToArray(); + } + + var output_loop_vars = outputs.Skip(first_loop_var_index).Take(num_flattened_oututs).ToArray(); + + if (!back_prop) + { + output_loop_vars = output_loop_vars.Select(t => array_ops.stop_gradient(t)).ToArray(); + } + outputs = _pack_sequence_as(loop_vars_signature, flat_orig_loop_vars, output_loop_vars); + + return outputs; + }); + } + + private static Tensors _tensor_array_to_flow(Tensors loop_vars) + { + if(loop_vars.NestType == NestType.Node) + { + if(loop_vars.NodeValue is FakeTensorByTensorArray fake) + { + return new Tensors(fake.TensorArray.flow); + } + else + { + return new Tensors(loop_vars.NodeValue!); + } + } + else if(loop_vars.NestType == NestType.List) + { + List> list = new(); + foreach(var item in loop_vars.ListValue!) + { + if(item.NestType == NestType.Node) + { + var nested = item.AsNest(); + if (nested.NodeValue is FakeTensorByTensorArray fake) + { + list.Add(new Nest(fake.TensorArray.flow)); + } + else + { + list.Add(new Nest(nested.NodeValue!)); + } + } + else + { + list.Add(new Nest(item.AsNest())); + } + } + return Tensors.FromNest(new Nest(list)); + } + else + { + throw new NotImplementedException(); + } + } + + private static Tensor[] _build_while_op(Tensor[] loop_vars, FuncGraph cond_graph, FuncGraph body_graph, + Shape[] output_shapes, int parallel_iterations, string name, int num_original_outputs, bool stateful_parallelism) + { + var cond_stateful_ops = cond_graph.get_operations().Select(x => x.op); + var body_stateful_ops = body_graph.get_operations().Select(x => x.op); + + bool is_stateful = cond_stateful_ops.Count() > 0 || body_stateful_ops.Count() > 0; + + Tensor[] _make_op(Tensor[] inputs) + { + Tensor[] outputs; + if (is_stateful) + { + outputs = gen_functional_ops._while( + inputs, + control_flow_util.create_new_tf_function(cond_graph), + control_flow_util.create_new_tf_function(body_graph), + output_shapes, + parallel_iterations, + name + ); + } + else + { + outputs = gen_functional_ops.stateless_while( + inputs, + control_flow_util.create_new_tf_function(cond_graph), + control_flow_util.create_new_tf_function(body_graph), + output_shapes, + parallel_iterations, + name + ); + } + var (while_op, tensors) = control_flow_util.get_op_and_outputs(outputs); + _copy_handle_data(body_graph.Outputs, tensors); + _set_read_only_resource_inputs_attr(while_op, new FuncGraph[]{cond_graph, body_graph}); + while_op._set_attr("_num_original_outputs", new AttrValue() { I = num_original_outputs }); + while_op._set_attr("_stateful_parallelism", new AttrValue() { B = stateful_parallelism }); + + cond_graph.outer_graph = ops.get_default_graph(); + body_graph.outer_graph = ops.get_default_graph(); + // TODO(Rinne): set the two graphs to while_op + return tensors; + } + + return control_flow_util.run_as_function_for_tape_gradients(_make_op, loop_vars); + } + + /// + /// Sets the list of resource inputs which are read-only. This is used by AutomaticControlDependencies. + /// + /// + /// + private static void _set_read_only_resource_inputs_attr(Operation op, FuncGraph[] branch_graphs) + { + List read_only_indices = Enumerable.Range(0, op.inputs.Length).ToList(); + foreach(var branch_graph in branch_graphs) + { + if (read_only_indices.Count == 0) + { + break; + } + var branch_read_only_indices = auto_control_deps_utils.get_read_only_resource_input_indices_graph(branch_graph); + read_only_indices = read_only_indices.Intersect(branch_read_only_indices).ToList(); + } + AttrValue.Types.ListValue listValue = new(); + listValue.I.AddRange(read_only_indices.OrderBy(x => x).Select(x => (long)x)); + op._set_attr(auto_control_deps_utils.READ_ONLY_RESOURCE_INPUTS_ATTR, new AttrValue() + { + List = listValue + }); + } + + private static Tensors _pack_sequence_as(INestStructure loop_vars_signature, Tensor[] flat_orig_loop_vars, Tensor[] loop_vars) + { + var flattened_loop_vars = zip(loop_vars, flat_orig_loop_vars).Select<(Tensor, Tensor), Tensor>(item => + { + var (flow, y) = item; + if (y is FakeTensorByTensorArray ta) + { + return new FakeTensorByTensorArray(tensor_array_ops.build_ta_with_new_flow(ta.TensorArray, flow)); + } + else + { + return flow; + } + }).ToArray(); + return Nest.PackSequenceAs(loop_vars_signature, flattened_loop_vars).ToTensors(); + } + + private static Tensor[] _get_intermediates(FuncGraph func_graph) + { + List intermediates = new(); + var reversed_captures = func_graph.captures.ToDictionary(x => x.Item2, x => x.Item1); + + foreach(var op in func_graph.get_operations()) + { + Debug.Assert(op is Operation); + var oper = (Operation)op; + if(oper.type == "Identity" || oper.type == "MutexLock") + { + continue; + } + foreach(var o in op.outputs) + { + if(o != func_graph.Inputs[0] && o.dtype != dtypes.resource && !reversed_captures.ContainsKey(o)) + { + intermediates.Add(o); + } + } + } + return intermediates.ToArray(); + } + + private static void _duplicate_body_captures_in_cond(FuncGraph cond_graph, Tensor[] body_graph_captures) + { + var types = body_graph_captures.Select(t => t.dtype).ToList(); + var c_graph = cond_graph.c_graph; + var placeholders = types.Select(x => CreatePlaceholder(c_graph, _build_cond_placeholders_name_prefix(cond_graph), x)).ToList(); + + var placeholder_ops = placeholders.Select(ph => new _OperationWithOutputs(ph.oper, cond_graph)).ToList(); + + List tensors = new(); + foreach(var (op, ph, dtype) in zip(placeholder_ops, placeholders, types)) + { + var tensor = Tensor._create_with_tf_output(op, 0, dtype, ph); + op._outputs = new Tensor[] { tensor }; + tensors.Add(tensor); + } + + var tuples = zip(body_graph_captures, tensors).ToList(); + var keys = body_graph_captures.Select(t => t.Id).ToList(); + cond_graph._captures.Update(zip(keys, tuples).ToDictionary(x => x.Item1, x => x.Item2)); + cond_graph.Inputs.AddRange(tensors); + } + + private static TF_Output CreatePlaceholder(SafeGraphHandle graph, string name, TF_DataType dtype) + { + var desc = c_api.TF_NewOperation(graph, "Placeholder", name); + c_api.TF_SetAttrType(desc, "dtype", dtype); + var op = c_api.TF_FinishOperation(desc, tf.Status); + tf.Status.Check(true); + var output = new TF_Output(); + output.oper = op; + output.index = 0; + return output; + } + + private static string _build_cond_placeholders_name_prefix(FuncGraph cond_graph) + { + return cond_graph.unique_name(cond_graph.Name + "___redundant_placeholder"); + } + + private static Tensor _convert_to_tensor_or_indexed_slices(Tensor value, TF_DataType dtype, + string name) + { + return ops.convert_to_tensor(value, dtype, name, false); + } + + private static Tensor _build_maximum_iterations_loop_var(int maximum_iterations = -1) + { + return ops.convert_to_tensor(maximum_iterations, dtypes.int32, "maximum_iterations"); + } + + private static void _copy_handle_data(IEnumerable src_tensors, IEnumerable dst_tensors) + { + foreach(var (src_t, dst_t) in zip(src_tensors, dst_tensors)) + { + handle_data_util.copy_handle_data(src_t, dst_t); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs index 498ffda7..e7ff9f74 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Creation.cs @@ -105,6 +105,13 @@ namespace Tensorflow _id = ops.uid(); } + internal static Tensor _create_with_tf_output(Operation op, int value_index, TF_DataType dtype, TF_Output tf_output) + { + Tensor ret = new Tensor(op, value_index, dtype); + ret._tf_output = tf_output; + return ret; + } + protected unsafe void InitTensor(Shape shape, TF_DataType dtype) { _handle = TF_NewTensor(shape, dtype, null); diff --git a/src/TensorFlowNET.Core/Tensors/TensorArray.cs b/src/TensorFlowNET.Core/Tensors/TensorArray.cs index fb59593c..ff74956a 100644 --- a/src/TensorFlowNET.Core/Tensors/TensorArray.cs +++ b/src/TensorFlowNET.Core/Tensors/TensorArray.cs @@ -14,7 +14,9 @@ limitations under the License. ******************************************************************************/ +using Tensorflow.Common.Types; using Tensorflow.Operations; +using static Tensorflow.Binding; namespace Tensorflow { @@ -44,5 +46,27 @@ namespace Tensorflow public abstract Tensor stack(string name = null); public abstract Tensor gather(Tensor indices, string name = null); + + internal bool _dynamic_size; + internal Tensor _size; + internal List _colocate_with; + internal Shape _element_shape; + + public static TensorArray Create(TF_DataType dtype, Tensor size = null, bool dynamic_size = false, + bool clear_after_read = true, string tensor_array_name = null, Tensor handle = null, Tensor flow = null, + bool infer_shape = true, Shape? element_shape = null, + bool colocate_with_first_write_call = true, string name = null) + { + if (tf.Context.executing_eagerly() && (flow is null || flow.dtype != dtypes.variant)) + { + return new _EagerTensorArray(dtype, size, dynamic_size, clear_after_read, tensor_array_name, handle, flow, + infer_shape, element_shape, colocate_with_first_write_call, name); + } + else + { + return new _GraphTensorArrayV2(dtype, size, dynamic_size, clear_after_read, tensor_array_name, handle, flow, + infer_shape, element_shape, colocate_with_first_write_call, name); + } + } } } diff --git a/src/TensorFlowNET.Core/Tensors/Tensors.cs b/src/TensorFlowNET.Core/Tensors/Tensors.cs index 259b1eec..38a3e5dc 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensors.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensors.cs @@ -4,6 +4,8 @@ using System.Collections; using System.Collections.Generic; using System.Linq; using Tensorflow.Common.Types; +using Tensorflow.Operations; +using Tensorflow.Common.Extensions; namespace Tensorflow { @@ -58,7 +60,7 @@ namespace Tensorflow public Tensor this[params string[] slices] => this.First()[slices]; - private Tensors(Nest nested) : base(nested) + internal Tensors(Nest nested) : base(nested) { } @@ -68,9 +70,9 @@ namespace Tensorflow } - public Tensors(IEnumerable tensors): base(tensors.Select(x => new Nest(x))) + public Tensors(IList tensors) : base(tensors.Select(x => new Nest(x))) { - + } public Tensors(NDArray nd): base(ops.convert_to_tensor(nd)) @@ -78,6 +80,32 @@ namespace Tensorflow } + /// + /// Get the element in shallow level. For example, for ts = [1, [2, 3], 4], + /// common indexer has ts[1] = 2. Shallow indexer has ts[1] = [2, 3] + /// + /// + /// + public Tensors GetShallow(int index) + { + if(NestType == NestType.Node) + { + if(index > 0) + { + throw new IndexOutOfRangeException(); + } + return this; + } + else if(NestType == NestType.List) + { + return ListValue![index].AsNest().ToTensors(); + } + else + { + throw new NotImplementedException(); + } + } + private static Nest DealWithConstructorArrayInput(Tensor[] tensors) { if (tensors.Length == 0) @@ -115,8 +143,8 @@ namespace Tensorflow else if(NestType == NestType.Node) { NestType = NestType.List; - ListValue = new() { new Nest(Value), new Nest(tensor) }; - Value = null; + ListValue = new() { new Nest(NodeValue), new Nest(tensor) }; + NodeValue = null; } else if(NestType == NestType.List) { @@ -125,7 +153,7 @@ namespace Tensorflow else //Empty { NestType = NestType.Node; - Value = tensor; + NodeValue = tensor; } } @@ -140,9 +168,9 @@ namespace Tensorflow else if (NestType == NestType.Node) { NestType = NestType.List; - ListValue = new() { new Nest(Value) }; + ListValue = new() { new Nest(NodeValue) }; ListValue.AddRange(tensors.Select(x => new Nest(x))); - Value = null; + NodeValue = null; } else if(NestType == NestType.List) { @@ -151,7 +179,7 @@ namespace Tensorflow else // empty { NestType = NestType.List; - ListValue = tensors.Select(x => new Nest(x)).ToList(); + ListValue = tensors.Select(x => new Nest(x) as INestStructure).ToList(); } } @@ -166,9 +194,9 @@ namespace Tensorflow else if(NestType == NestType.Node) { NestType = NestType.List; - ListValue = new() { new Nest(Value) }; + ListValue = new() { new Nest(NodeValue) }; ListValue.Insert(index, new Nest(tensor)); - Value = null; + NodeValue = null; } else { @@ -283,7 +311,7 @@ namespace Tensorflow => tensors?.SingleOrNull; public static implicit operator Tensor[](Tensors tensors) - => tensors.Flatten().ToArray(); + => tensors.Flatten().ToArray(); #endregion public static Tensors? FromNest(Nest nested) @@ -298,7 +326,7 @@ namespace Tensorflow public void Deconstruct(out Tensor a, out Tensors? b) { a = this.First(); - b = Length == 1? null : new Tensors(this.Skip(1)); + b = Length == 1? null : new Tensors(this.Skip(1).ToArray()); } public override string ToString() diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 6d1385ca..fb9bccf3 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -576,7 +576,7 @@ namespace Tensorflow public static HandleData get_resource_handle_data(Tensor graph_op) { var handle_data = c_api.TFC_GetHandleShapeAndType(graph_op.graph.c_graph, graph_op._as_tf_output()); - return HandleData.Parser.ParseFrom(tf.compat.as_bytes(c_api.StringPiece(handle_data))); + return HandleData.Parser.ParseFrom(c_api.ByteStringPiece(handle_data)); } public static void dismantle_graph(Graph graph) diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 1336e9af..8dbcf90d 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -25,6 +25,7 @@ using static Tensorflow.Binding; using static Tensorflow.Graphs.SubGraphUtility; using Tensorflow.Util; using Tensorflow.Common.Types; +using System.Diagnostics; namespace Tensorflow.Keras { @@ -485,7 +486,7 @@ namespace Tensorflow.Keras var first_flatted_input = flatted_inptus[0]; var time_steps = first_flatted_input.shape[0]; var batch = first_flatted_input.shape[1]; - var time_steps_t = (int)first_flatted_input.shape[0]; + var time_steps_t = tf.shape(first_flatted_input)[0]; foreach (var input_ in flatted_inptus) { @@ -704,7 +705,7 @@ namespace Tensorflow.Keras var input_ta = new List(); for (int i = 0; i < flatted_inptus.Count; i++) { - input_ta.Add(tf.TensorArray(dtype: flatted_inptus[i].dtype, size: time_steps_t)); + input_ta.Add(TensorArray.Create(dtype: flatted_inptus[i].dtype, size: time_steps_t)); } foreach(var (ta, input_) in zip(input_ta, flatted_inptus)) @@ -730,18 +731,15 @@ namespace Tensorflow.Keras (output_time_zero, _) = step_function(input_time_zero, constants is null ? initial_states : initial_states.MergeWith(constants)); - int output_ta_size = return_all_outputs ? time_steps_t : 1; + Tensor output_ta_size = return_all_outputs ? time_steps_t : constant_op.constant(1); var output_ta = new List(); - for (int i = 0; i < output_time_zero.ToList().Count; i++) + foreach(var output in output_time_zero.Flatten()) { - var Out = output_time_zero.ToList()[i]; - output_ta.Add(tf.TensorArray(dtype: Out.dtype, size: output_ta_size, element_shape: Out.shape)); + output_ta.Add(TensorArray.Create(dtype: output.dtype, size: output_ta_size, element_shape: output.shape)); } var time = tf.constant(0, dtype: TF_DataType.TF_INT32, name: "time"); - - Func? masking_fn; Func? compute_masked_output = null; if (mask != null) @@ -750,7 +748,7 @@ namespace Tensorflow.Keras { mask = tf.reverse(mask, axis: new[] { 0 }); } - var mask_ta = tf.TensorArray(dtype: TF_DataType.TF_BOOL, size: time_steps_t); + var mask_ta = TensorArray.Create(dtype: TF_DataType.TF_BOOL, size: time_steps_t); mask_ta = mask_ta.unstack(mask); masking_fn = (time) => @@ -810,9 +808,9 @@ namespace Tensorflow.Keras masking_fn = null; } - Func cond = (time) => (time < time_steps_t); + Func cond = (time) => (time[0] < time_steps_t); int parallel_iterations = 32; - new_states = states; + Tensors final_outputs; if (masking_fn != null) { // Mask for the T output will be base on the output of T - 1. In the @@ -825,7 +823,7 @@ namespace Tensorflow.Keras var prev_output = flat_zero_output; var output_ta_t = output_ta; - Tensor _step(Tensor time) + Tensors _step(Tensors tensors) { /* RNN step function. @@ -838,23 +836,28 @@ namespace Tensorflow.Keras Tuple(todo): `(time + 1, output_ta_t, output) + tuple(new_states)` */ + Tensor time = tensors[0]; + TensorArray output_ta_t = (tensors[1] as FakeTensorByTensorArray).TensorArray; + Tensors prev_output = tensors.GetShallow(2); + Tensors states = new Tensors(tensors.Skip(2 + prev_output.Length).ToArray()); + var flat_current_input = input_ta.Select(x => x.read(time)).ToList(); // maybe set shape // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type var current_input = Nest.PackSequenceAs(inputs, flat_current_input).ToTensors(); var mask_t = masking_fn(time); - var (output, new_states_internal) = step_function(current_input, new_states.MergeWith(constants)); + var (output, new_states) = step_function(current_input, states.MergeWith(constants)); // mask output var flat_output = Nest.Flatten(output).ToList(); - var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.ToList(); + var flat_mask_output = zero_output_for_mask ? flat_zero_output : prev_output.Flatten().ToList(); // TODO(Wanglongzhi2001),deal with compute_masked_output's third parameter's type var flat_new_output = compute_masked_output(mask_t, flat_output, flat_mask_output); // mask states - var flat_state = states.ToList(); - var flat_new_state = new_states_internal.ToList(); + var flat_state = states.Flatten().ToList(); + var flat_new_state = new_states.Flatten().ToList(); foreach (var (state, new_state) in zip(flat_state, flat_new_state)) { @@ -865,38 +868,37 @@ namespace Tensorflow.Keras } var flat_final_state = compute_masked_output(mask_t, flat_new_state, flat_state); - new_states_internal = Nest.PackSequenceAs(new_states, flat_final_state).ToTensors(); + new_states = Nest.PackSequenceAs(new_states, flat_final_state.ToArray()).ToTensors(); var ta_index_to_write = return_all_outputs ? time : tf.constant(0); - output_ta_t = zip(output_ta_t, flat_new_output).Select(item => - { - var (ta, out_) = item; - return ta.write(ta_index_to_write, out_); - }).ToList(); + Debug.Assert(flat_output.Count() == 1); + output_ta_t = output_ta_t.write(ta_index_to_write, flat_new_output.First()); - - new_states_internal = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); - - output_ta = output_ta_t; - new_states = new_states_internal; - return time + 1; + return new Tensor[] { time + 1, new FakeTensorByTensorArray(output_ta_t) }.Concat(flat_new_output).Concat(new_states) + .ToArray().ToTensors(); } - var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: time, parallel_iterations: parallel_iterations); + var loop_vars = new Tensor[] { time + 1, new FakeTensorByTensorArray(output_ta[0]) } + .Concat(flat_zero_output.Flatten()).Concat(states).ToArray().ToTensors(); + final_outputs = control_flow_ops.while_loop(cond: cond, body: _step, loop_vars: loop_vars, parallel_iterations: parallel_iterations); + new_states = final_outputs.Skip(3).ToList(); } else { var output_ta_t = output_ta; new_states = states; - Tensor _step(Tensor time) + Tensors _step(Tensors tensors) { + Tensor time = tensors[0]; + TensorArray output_ta_t = (tensors[1] as FakeTensorByTensorArray).TensorArray; + Tensors states = new Tensors(tensors.Skip(2).ToArray()); var flat_current_input = input_ta.Select(x => x.read(time)).ToList(); // maybe set shape // TODO(Wanglongzhi2001),deal with nest.pack_sequence_as's return type var current_input = Nest.PackSequenceAs(inputs, flat_current_input).ToTensors(); - var (output, new_states_internal) = step_function(current_input, new_states.MergeWith(constants)); + var (output, new_states) = step_function(current_input, states.MergeWith(constants)); var flat_state = new_states.Flatten().ToList(); - var flat_new_state = new_states_internal.Flatten().ToList(); + var flat_new_state = new_states.Flatten().ToList(); foreach (var (state, new_state) in zip(flat_state, flat_new_state)) { if (new_state is Tensor) @@ -906,24 +908,23 @@ namespace Tensorflow.Keras } var flat_output = Nest.Flatten(output); var ta_index_to_write = return_all_outputs ? time : tf.constant(0); - output_ta_t = zip(output_ta_t, flat_output).Select(item => - { - var (ta, out_) = item; - return ta.write(ta_index_to_write, out_); - }).ToList(); - - new_states_internal = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); - output_ta = output_ta_t; - new_states = new_states_internal; - return time + 1; + Debug.Assert(flat_output.Count() == 1); + output_ta_t = output_ta_t.write(ta_index_to_write, flat_output.First()); + + new_states = Nest.PackSequenceAs(initial_states, flat_new_state).ToTensors(); + return new Tensor[] { time + 1, new FakeTensorByTensorArray(output_ta_t) }.Concat(new_states).ToArray().ToTensors(); } - var final_outputs = tf.while_loop(cond: cond, body: _step, loop_vars: time, parallel_iterations: parallel_iterations); + Debug.Assert(output_ta.Count == 1); + var loop_vars = new Tensor[] { time + 1, new FakeTensorByTensorArray(output_ta[0]) }.Concat(states).ToArray().ToTensors(); + final_outputs = control_flow_ops.while_loop(cond: cond, body: _step, loop_vars: loop_vars, parallel_iterations: parallel_iterations); + new_states = final_outputs.Skip(2).ToList(); } - outputs = outputs.MergeWith(output_ta.Select(o => o.stack()).ToTensors()); - last_output = last_output.MergeWith(outputs.Select(o => o[-1]).ToTensors()); - outputs = Nest.PackSequenceAs(output_time_zero, outputs).ToTensors(); - last_output = Nest.PackSequenceAs(output_time_zero, last_output).ToTensors(); + output_ta = new List { (final_outputs[1] as FakeTensorByTensorArray).TensorArray }; + outputs = outputs.MergeWith(output_ta.Select(o => o.stack()).ToArray().ToTensors()); + last_output = last_output.MergeWith(outputs.Select(o => o[-1]).ToArray().ToTensors()); + outputs = Nest.PackSequenceAs(output_time_zero, (Tensor[])outputs).ToTensors(); + last_output = Nest.PackSequenceAs(output_time_zero, (Tensor[])last_output).ToTensors(); } Func set_shape; diff --git a/src/TensorFlowNET.Keras/Engine/Model.Build.cs b/src/TensorFlowNET.Keras/Engine/Model.Build.cs index 69afdef9..23336383 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Build.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Build.cs @@ -23,7 +23,7 @@ namespace Tensorflow.Keras.Engine var graph = tf.executing_eagerly() ? new FuncGraph("build_graph") : keras.backend.get_graph(); graph.as_default(); var shapes = input_shape.ToShapeArray(); - var x = new Tensors(shapes.Select(x => base_layer_utils.generate_placeholders_from_shape(x))); + var x = new Tensors(shapes.Select(x => base_layer_utils.generate_placeholders_from_shape(x)).ToArray()); try { Call(x, training: false); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 185de4f4..d807b204 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -95,7 +95,7 @@ namespace Tensorflow.Keras.Engine { var data_handler = new DataHandler(new DataHandlerArgs { - X = new Tensors(x), + X = new Tensors(x.ToArray()), Y = y, Model = this, StepsPerExecution = _steps_per_execution @@ -188,7 +188,7 @@ namespace Tensorflow.Keras.Engine { var data = iterator.next(); var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = train_step(data_handler, new Tensors(data.Take(x_size)), new Tensors(data.Skip(x_size))); + var outputs = train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())); tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index bb8e18cc..76c592ad 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -110,7 +110,7 @@ namespace Tensorflow.Keras.Engine var data_handler = new DataHandler(new DataHandlerArgs { - X = new Tensors(train_x), + X = new Tensors(train_x.ToArray()), Y = train_y, BatchSize = batch_size, InitialEpoch = initial_epoch, diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index 905ea453..48c16e18 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -21,7 +21,7 @@ namespace Tensorflow.Keras.Engine { var data = iterator.next(); var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = train_step(data_handler, new Tensors(data.Take(x_size)), new Tensors(data.Skip(x_size))); + var outputs = train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())); tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 78d3dac9..d2669ccc 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -4,10 +4,11 @@ using System.Text; using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Layers.Rnn { - public abstract class DropoutRNNCellMixin: RnnCellBase + public abstract class DropoutRNNCellMixin: Layer, IRnnCell { public float dropout; public float recurrent_dropout; @@ -17,6 +18,14 @@ namespace Tensorflow.Keras.Layers.Rnn } + public abstract GeneralizedTensorShape StateSize { get; } + public abstract GeneralizedTensorShape OutputSize { get; } + public abstract bool SupportOptionalArgs { get; } + public virtual Tensors GetInitialState(Tensors inputs, Tensor batch_size, TF_DataType dtype) + { + return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); + } + protected void _create_non_trackable_mask_cache() { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0ebd7362..77f7d927 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -206,7 +206,6 @@ namespace Tensorflow.Keras.Layers.Rnn // append bacth dim state_spec_shape = new int[] { -1 }.concat(state_spec_shape); return new InputSpec(shape: state_spec_shape); - } // Check whether the input shape contains any nested shapes. It could be @@ -298,7 +297,7 @@ namespace Tensorflow.Keras.Layers.Rnn // cell_call_fn = (self.cell.__call__ if callable(self.cell) else self.cell.call) Func step; - bool is_tf_rnn_cell = _cell.IsTFRnnCell; + bool is_tf_rnn_cell = false; if (constants is not null) { if (!_cell.SupportOptionalArgs) @@ -310,8 +309,8 @@ namespace Tensorflow.Keras.Layers.Rnn step = (inputs, states) => { - constants = new Tensors(states.TakeLast(_num_constants)); - states = new Tensors(states.SkipLast(_num_constants)); + constants = new Tensors(states.TakeLast(_num_constants).ToArray()); + states = new Tensors(states.SkipLast(_num_constants).ToArray()); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); return (output, new_states.Single); @@ -395,12 +394,12 @@ namespace Tensorflow.Keras.Layers.Rnn { if (_num_constants != 0) { - initial_state = new Tensors(inputs.Skip(1)); + initial_state = new Tensors(inputs.Skip(1).ToArray()); } else { - initial_state = new Tensors(inputs.Skip(1).SkipLast(_num_constants)); - constants = new Tensors(inputs.TakeLast(_num_constants)); + initial_state = new Tensors(inputs.Skip(1).SkipLast(_num_constants).ToArray()); + constants = new Tensors(inputs.TakeLast(_num_constants).ToArray()); } if (len(initial_state) == 0) initial_state = null; @@ -558,36 +557,14 @@ namespace Tensorflow.Keras.Layers.Rnn protected Tensors get_initial_state(Tensors inputs) { - var get_initial_state_fn = _cell.GetType().GetMethod("get_initial_state"); - var input = inputs[0]; - var input_shape = inputs.shape; + var input_shape = array_ops.shape(inputs); var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; var dtype = input.dtype; - Tensors init_state = new Tensors(); - - if(get_initial_state_fn != null) - { - init_state = (Tensors)get_initial_state_fn.Invoke(_cell, new object[] { inputs, batch_size, dtype }); - - } - //if (_cell is RnnCellBase rnn_base_cell) - //{ - // init_state = rnn_base_cell.GetInitialState(null, batch_size, dtype); - //} - else - { - init_state = RnnUtils.generate_zero_filled_state(batch_size, _cell.StateSize, dtype); - } + Tensors init_state = _cell.GetInitialState(null, batch_size, dtype); return init_state; } - - // Check whether the state_size contains multiple states. - public static bool is_multiple_state(GeneralizedTensorShape state_size) - { - return state_size.Shapes.Length > 1; - } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs deleted file mode 100644 index 751312e5..00000000 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RnnCellBase.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; -using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Utils; - -namespace Tensorflow.Keras.Layers.Rnn -{ - public abstract class RnnCellBase: Layer, IRnnCell - { - public RnnCellBase(LayerArgs args) : base(args) { } - public abstract GeneralizedTensorShape StateSize { get; } - public abstract GeneralizedTensorShape OutputSize { get; } - public abstract bool IsTFRnnCell { get; } - public abstract bool SupportOptionalArgs { get; } - public virtual Tensors GetInitialState(Tensors inputs, long batch_size, TF_DataType dtype) - { - return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); - } - } -} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 39610ff5..3b4b9419 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -7,6 +7,7 @@ using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; using Tensorflow.Keras.Utils; +using Tensorflow.Graphs; namespace Tensorflow.Keras.Layers.Rnn { @@ -28,7 +29,6 @@ namespace Tensorflow.Keras.Layers.Rnn public override GeneralizedTensorShape StateSize => _state_size; public override GeneralizedTensorShape OutputSize => _output_size; - public override bool IsTFRnnCell => true; public override bool SupportOptionalArgs => false; public SimpleRNNCell(SimpleRNNCellArgs args) : base(args) @@ -98,7 +98,6 @@ namespace Tensorflow.Keras.Layers.Rnn { prev_output = math_ops.multiply(prev_output, rec_dp_mask); } - var tmp = _recurrent_kernel.AsTensor(); Tensor output = h + math_ops.matmul(prev_output, _recurrent_kernel.AsTensor()); if (_args.Activation != null) @@ -117,9 +116,9 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) + public Tensors get_initial_state(Tensors inputs = null, Tensor batch_size = null, TF_DataType dtype = TF_DataType.DtInvalid) { - return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size.Value, dtype.Value); + return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 56634853..fb74d6d2 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -15,7 +15,7 @@ namespace Tensorflow.Keras.Layers.Rnn public class StackedRNNCells : Layer, IRnnCell { public IList Cells { get; set; } - public bool reverse_state_order; + public bool _reverse_state_order; public StackedRNNCells(StackedRNNCellsArgs args) : base(args) { @@ -23,22 +23,11 @@ namespace Tensorflow.Keras.Layers.Rnn { args.Kwargs = new Dictionary(); } - foreach (var cell in args.Cells) - { - //Type type = cell.GetType(); - //var CallMethodInfo = type.GetMethod("Call"); - //if (CallMethodInfo == null) - //{ - // throw new ValueError( - // "All cells must have a `Call` method. " + - // $"Received cell without a `Call` method: {cell}"); - //} - } Cells = args.Cells; - reverse_state_order = (bool)args.Kwargs.Get("reverse_state_order", false); + _reverse_state_order = (bool)args.Kwargs.Get("reverse_state_order", false); - if (reverse_state_order) + if (_reverse_state_order) { throw new WarningException("reverse_state_order=True in StackedRNNCells will soon " + "be deprecated. Please update the code to work with the " + @@ -47,49 +36,37 @@ namespace Tensorflow.Keras.Layers.Rnn } } + public bool SupportOptionalArgs => false; + public GeneralizedTensorShape StateSize { get { - GeneralizedTensorShape state_size = new GeneralizedTensorShape(1, Cells.Count); - if (reverse_state_order && Cells.Count > 0) + if (_reverse_state_order) { - var idxAndCell = Cells.Reverse().Select((cell, idx) => (idx, cell)); - foreach (var cell in idxAndCell) - { - state_size.Shapes[cell.idx] = cell.cell.StateSize.Shapes.First(); - } + var state_sizes = Cells.Reverse().Select(cell => cell.StateSize); + return new GeneralizedTensorShape(new Nest(state_sizes.Select(s => new Nest(s)))); } else { - //foreach (var cell in Cells) - //{ - // state_size.Shapes.add(cell.StateSize.Shapes.First()); - - //} - var idxAndCell = Cells.Select((cell, idx) => (idx, cell)); - foreach (var cell in idxAndCell) - { - state_size.Shapes[cell.idx] = cell.cell.StateSize.Shapes.First(); - } + var state_sizes = Cells.Select(cell => cell.StateSize); + return new GeneralizedTensorShape(new Nest(state_sizes.Select(s => new Nest(s)))); } - return state_size; } } - public object output_size + public GeneralizedTensorShape OutputSize { get { - var lastCell = Cells.LastOrDefault(); - if (lastCell.OutputSize.ToSingleShape() != -1) + var lastCell = Cells.Last(); + if(lastCell.OutputSize is not null) { return lastCell.OutputSize; } - else if (RNN.is_multiple_state(lastCell.StateSize)) + else if (RnnUtils.is_multiple_state(lastCell.StateSize)) { return lastCell.StateSize.First(); - //throw new NotImplementedException(""); } else { @@ -98,79 +75,65 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) + public Tensors GetInitialState(Tensors inputs = null, Tensor batch_size = null, TF_DataType dtype = TF_DataType.DtInvalid) { - var cells = reverse_state_order ? Cells.Reverse() : Cells; - Tensors initial_states = new Tensors(); + var cells = _reverse_state_order ? Cells.Reverse() : Cells; + List initial_states = new List(); foreach (var cell in cells) { - var get_initial_state_fn = cell.GetType().GetMethod("get_initial_state"); - if (get_initial_state_fn != null) - { - var result = (Tensors)get_initial_state_fn.Invoke(cell, new object[] { inputs, batch_size, dtype }); - initial_states.Add(result); - } - else - { - initial_states.Add(RnnUtils.generate_zero_filled_state_for_cell(cell, inputs, batch_size.Value, dtype.Value)); - } + initial_states.Add(cell.GetInitialState(inputs, batch_size, dtype)); } - return initial_states; + return new Tensors(initial_states); } - protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) { // Recover per-cell states. - var state_size = reverse_state_order ? StateSize.Reverse() : StateSize; - var nested_states = reverse_state_order ? state.Flatten().Reverse() : state.Flatten(); + var state_size = _reverse_state_order ? new GeneralizedTensorShape(StateSize.Reverse()) : StateSize; + var nested_states = Nest.PackSequenceAs(state_size, Nest.Flatten(states).ToArray()); - - var new_nest_states = new Tensors(); + var new_nest_states = Nest.Empty; // Call the cells in order and store the returned states. - foreach (var (cell, states) in zip(Cells, nested_states)) + foreach (var (cell, internal_states) in zip(Cells, nested_states)) { - // states = states if tf.nest.is_nested(states) else [states] - var type = cell.GetType(); - bool IsTFRnnCell = type.GetProperty("IsTFRnnCell") != null; - state = len(state) == 1 && IsTFRnnCell ? state.FirstOrDefault() : state; - RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; Tensors? constants = rnn_optional_args?.Constants; Tensors new_states; - (inputs, new_states) = cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); + (inputs, new_states) = cell.Apply(inputs, internal_states, optional_args: new RnnOptionalArgs() { Constants = constants }); - new_nest_states.Add(new_states); + new_nest_states = new_nest_states.MergeWith(new_states); } - new_nest_states = reverse_state_order ? new_nest_states.Reverse().ToArray() : new_nest_states.ToArray(); - return new Nest(new List> { - new Nest(new List> { new Nest(inputs.Single()) }), new Nest(new_nest_states) }) - .ToTensors(); + return Tensors.FromNest((inputs, Nest.PackSequenceAs(state_size, Nest.Flatten(new_nest_states).ToArray()))); } - - - public void build() + public override void build(KerasShapesWrapper input_shape) { - built = true; - // @tf_utils.shape_type_conversion - // def build(self, input_shape) : - // if isinstance(input_shape, list) : - // input_shape = input_shape[0] - // for cell in self.cells: - // if isinstance(cell, Layer) and not cell.built: - // with K.name_scope(cell.name): - // cell.build(input_shape) - // cell.built = True - // if getattr(cell, 'output_size', None) is not None: - // output_dim = cell.output_size - // elif _is_multiple_state(cell.state_size) : - // output_dim = cell.state_size[0] - // else: - // output_dim = cell.state_size - // input_shape = tuple([input_shape[0]] + - // tensor_shape.TensorShape(output_dim).as_list()) - // self.built = True + var shape = input_shape.ToSingleShape(); + foreach(var cell in Cells) + { + if(cell is Layer layer && !layer.Built) + { + // ignored the name scope. + layer.build(shape); + layer.Built = true; + } + GeneralizedTensorShape output_dim; + if(cell.OutputSize is not null) + { + output_dim = cell.OutputSize; + } + else if (RnnUtils.is_multiple_state(cell.StateSize)) + { + output_dim = cell.StateSize.First(); + } + else + { + output_dim = cell.StateSize; + } + shape = new Shape(new long[] { shape.dims[0] }.Concat(output_dim.ToSingleShape().dims).ToArray()); + } + this.Built = true; } public override IKerasConfig get_config() @@ -198,14 +161,5 @@ namespace Tensorflow.Keras.Layers.Rnn // deserialize_layer(cell_config, custom_objects = custom_objects)) // return cls(cells, **config) } - - public (Tensor, Tensors) Call(Tensors inputs, Tensors states, bool? training = null) - { - throw new NotImplementedException(); - } - - public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); - public bool IsTFRnnCell => true; - public bool SupportOptionalArgs => throw new NotImplementedException(); } } diff --git a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs index 3109eb77..7ff3f9fb 100644 --- a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs +++ b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs @@ -10,20 +10,21 @@ namespace Tensorflow.Keras.Utils { internal static class RnnUtils { - internal static Tensors generate_zero_filled_state(long batch_size_tensor, GeneralizedTensorShape state_size, TF_DataType dtype) + internal static Tensors generate_zero_filled_state(Tensor batch_size_tensor, GeneralizedTensorShape state_size, TF_DataType dtype) { Func create_zeros; create_zeros = (GeneralizedTensorShape unnested_state_size) => { var flat_dims = unnested_state_size.ToSingleShape().dims; - var init_state_size = new long[] { batch_size_tensor }.Concat(flat_dims).ToArray(); - return array_ops.zeros(new Shape(init_state_size), dtype: dtype); + var init_state_size = new Tensor[] { batch_size_tensor }. + Concat(flat_dims.Select(x => tf.constant(x, dtypes.int32))).ToArray(); + return array_ops.zeros(init_state_size, dtype: dtype); }; // TODO(Rinne): map structure with nested tensors. - if(state_size.Shapes.Length > 1) + if(state_size.TotalNestedCount > 1) { - return new Tensors(state_size.ToShapeArray().Select(s => create_zeros(new GeneralizedTensorShape(s)))); + return new Tensors(state_size.Flatten().Select(s => create_zeros(new GeneralizedTensorShape(s))).ToArray()); } else { @@ -32,11 +33,11 @@ namespace Tensorflow.Keras.Utils } - internal static Tensors generate_zero_filled_state_for_cell(IRnnCell cell, Tensors inputs, long batch_size, TF_DataType dtype) + internal static Tensors generate_zero_filled_state_for_cell(IRnnCell cell, Tensors inputs, Tensor batch_size, TF_DataType dtype) { - if (inputs != null) + if (inputs is not null) { - batch_size = inputs.shape[0]; + batch_size = array_ops.shape(inputs)[0]; dtype = inputs.dtype; } return generate_zero_filled_state(batch_size, cell.StateSize, dtype); @@ -77,17 +78,27 @@ namespace Tensorflow.Keras.Utils Debug.Assert(initial_state is null && constants is null); if(num_constants > 0) { - constants = inputs.TakeLast(num_constants).ToTensors(); - inputs = inputs.SkipLast(num_constants).ToTensors(); + constants = inputs.TakeLast(num_constants).ToArray().ToTensors(); + inputs = inputs.SkipLast(num_constants).ToArray().ToTensors(); } if(inputs.Length > 1) { - initial_state = inputs.Skip(1).ToTensors(); - inputs = inputs.Take(1).ToTensors(); + initial_state = inputs.Skip(1).ToArray().ToTensors(); + inputs = inputs.Take(1).ToArray().ToTensors(); } } return (inputs, initial_state, constants); } + + /// + /// Check whether the state_size contains multiple states. + /// + /// + /// + public static bool is_multiple_state(GeneralizedTensorShape state_size) + { + return state_size.TotalNestedCount > 1; + } } } diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/ControlFlowApiTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/ControlFlowApiTest.cs index 6d7182e0..23dc1d44 100644 --- a/test/TensorFlowNET.UnitTest/ManagedAPI/ControlFlowApiTest.cs +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/ControlFlowApiTest.cs @@ -28,8 +28,8 @@ namespace TensorFlowNET.UnitTest.ManagedAPI var i = tf.constant(2); var j = tf.constant(3); - Func c = (x) => tf.less(x[0] + x[1], 10); - Func b = (x) => new[] { tf.add(x[0], 1), tf.add(x[1], 1) }; + Func c = (x) => tf.less(x[0] + x[1], 10); + Func b = (x) => new[] { tf.add(x[0], 1), tf.add(x[1], 1) }; var r = tf.while_loop(c, b, new[] { i, j }); Assert.AreEqual(5, (int)r[0]); Assert.AreEqual(6, (int)r[1]); diff --git a/tools/Tensorflow.CodeGen/FunctionGenerator.cs b/tools/Tensorflow.CodeGen/FunctionGenerator.cs index 93f9ea4e..186e6a27 100644 --- a/tools/Tensorflow.CodeGen/FunctionGenerator.cs +++ b/tools/Tensorflow.CodeGen/FunctionGenerator.cs @@ -21,7 +21,8 @@ namespace Tensorflow.CodeGen { sb.Append("Operation "); } - else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr)) + else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr) + && string.IsNullOrEmpty(op.OutputArg[0].TypeListAttr)) { sb.Append("Tensor "); } @@ -70,7 +71,8 @@ namespace Tensorflow.CodeGen { sb.AppendLine("return null;"); } - else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr)) + else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr) + && string.IsNullOrEmpty(op.OutputArg[0].TypeListAttr)) { sb.AppendLine("return _fast_path_result[0];"); } @@ -149,7 +151,8 @@ namespace Tensorflow.CodeGen { sb.AppendLine("return _op;"); } - else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr)) + else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr) + && string.IsNullOrEmpty(op.OutputArg[0].TypeListAttr)) { sb.AppendLine("return _result[0];"); } @@ -174,7 +177,7 @@ namespace Tensorflow.CodeGen { argName = $"{argName}_"; } - if (!string.IsNullOrEmpty(arg.NumberAttr)) + if (!string.IsNullOrEmpty(arg.NumberAttr) || !string.IsNullOrEmpty(arg.TypeListAttr)) { sb.Append($"Tensors {argName}, "); } @@ -273,7 +276,8 @@ namespace Tensorflow.CodeGen { sb.Append("Operation "); } - else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr)) + else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr) + && string.IsNullOrEmpty(op.OutputArg[0].TypeListAttr)) { sb.Append("Tensor "); } @@ -366,6 +370,13 @@ namespace Tensorflow.CodeGen sb.Append($"\"{attr.Name}\", {attrRealName}, "); } } + else if(attr.Type == "list(type)") + { + if (op.InputArg.Any(x => x.TypeListAttr == attr.Name)) + { + continue; + } + } else if(attr.Type == "int" && op.InputArg.Any(x => x.NumberAttr == attr.Name)) { bool found = false; @@ -408,7 +419,8 @@ namespace Tensorflow.CodeGen { sb.AppendLine("return null;"); } - else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr)) + else if (outputArgsCount == 1 && string.IsNullOrEmpty(op.OutputArg[0].NumberAttr) + && string.IsNullOrEmpty(op.OutputArg[0].TypeListAttr)) { sb.AppendLine("return _result[0];"); } diff --git a/tools/Tensorflow.CodeGen/Program.cs b/tools/Tensorflow.CodeGen/Program.cs index f9d44ce8..cea52e0b 100644 --- a/tools/Tensorflow.CodeGen/Program.cs +++ b/tools/Tensorflow.CodeGen/Program.cs @@ -5,7 +5,7 @@ using System.Text; using System.Xml.Linq; using Tensorflow.CodeGen; -GenOpsWriter writer = new(@"D:\development\tf.net\gen_ops", +GenOpsWriter writer = new(@"D:\development\tf.net\gen_ops_v2", @"D:\Apps\miniconda3\envs\tf2.11\Lib\site-packages\tensorflow\python\ops", @"D:\development\tf.net\tensorflow-2.11.0\tensorflow\core\api_def\base_api", @"D:\development\tf.net\tensorflow-2.11.0\tensorflow\core\ops\ops.pbtxt"); diff --git a/tools/Tensorflow.CodeGen/Utils.cs b/tools/Tensorflow.CodeGen/Utils.cs index d3f30d9f..19de6c0e 100644 --- a/tools/Tensorflow.CodeGen/Utils.cs +++ b/tools/Tensorflow.CodeGen/Utils.cs @@ -155,6 +155,10 @@ namespace Tensorflow.CodeGen } else if (attr.Type == "list(type)") { + if(op.InputArg.Any(x => x.TypeListAttr == attr.Name)) + { + continue; + } if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.Type) { List values = new(); @@ -231,11 +235,11 @@ namespace Tensorflow.CodeGen } else if (attr.Type == "func") { - res.Add((attr.Name, "Func", "NOVALUE")); + res.Add((attr.Name, "object", "NOVALUE")); } else if (attr.Type == "list(func)") { - res.Add((attr.Name, "Func[]", "NOVALUE")); + res.Add((attr.Name, "object[]", "NOVALUE")); } else if (attr.Type == "tensor") { From 07ea65683362cc2a633e9de0a7e0b550794d2474 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 16 Jun 2023 16:15:01 +0800 Subject: [PATCH 08/77] fix: error when training SimpleRNN. --- .../Exceptions/NotOkStatusException.cs | 19 +++++++ .../Operations/Operation.cs | 11 +++- .../Operations/gen_math_ops.cs | 3 +- src/TensorFlowNET.Core/Status/Status.cs | 3 +- src/TensorFlowNET.Keras/IsExternalInit.cs | 4 ++ src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 54 ++++++++++++------- .../Layers/Rnn/SimpleRNN.cs | 14 ----- .../Layers/Rnn.Test.cs | 5 ++ 8 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 src/TensorFlowNET.Core/Exceptions/NotOkStatusException.cs create mode 100644 src/TensorFlowNET.Keras/IsExternalInit.cs diff --git a/src/TensorFlowNET.Core/Exceptions/NotOkStatusException.cs b/src/TensorFlowNET.Core/Exceptions/NotOkStatusException.cs new file mode 100644 index 00000000..c283c1a4 --- /dev/null +++ b/src/TensorFlowNET.Core/Exceptions/NotOkStatusException.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Exceptions +{ + public class NotOkStatusException : TensorflowException + { + public NotOkStatusException() : base() + { + + } + + public NotOkStatusException(string message) : base(message) + { + + } + } +} diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs index 5e689c65..d31b26d4 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.cs @@ -186,7 +186,16 @@ namespace Tensorflow } public virtual T get_attr(string name) - => (T)get_attr(name); + { + if (typeof(T).IsValueType) + { + return (T)Convert.ChangeType(get_attr(name), typeof(T)); + } + else + { + return (T)get_attr(name); + } + } internal unsafe TF_DataType _get_attr_type(string name) { diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index 3456d9b3..6eb7a411 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -4633,8 +4633,9 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatMul", name) { args = new object[] { a, b }, attrs = new Dictionary() { ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b } }); return _fast_path_result[0]; } - catch (Exception) + catch (Exception ex) { + Console.WriteLine(); } try { diff --git a/src/TensorFlowNET.Core/Status/Status.cs b/src/TensorFlowNET.Core/Status/Status.cs index a890c2ae..12b6fba2 100644 --- a/src/TensorFlowNET.Core/Status/Status.cs +++ b/src/TensorFlowNET.Core/Status/Status.cs @@ -17,6 +17,7 @@ using System; using System.Diagnostics; using System.Runtime.CompilerServices; +using Tensorflow.Exceptions; using Tensorflow.Util; using static Tensorflow.c_api; @@ -88,7 +89,7 @@ namespace Tensorflow case TF_Code.TF_INVALID_ARGUMENT: throw new InvalidArgumentError(message); default: - throw new TensorflowException(message); + throw new NotOkStatusException(message); } } } diff --git a/src/TensorFlowNET.Keras/IsExternalInit.cs b/src/TensorFlowNET.Keras/IsExternalInit.cs new file mode 100644 index 00000000..11f062fa --- /dev/null +++ b/src/TensorFlowNET.Keras/IsExternalInit.cs @@ -0,0 +1,4 @@ +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit { } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 77f7d927..f99bc23a 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -11,6 +11,7 @@ using Tensorflow.Common.Extensions; using System.Linq.Expressions; using Tensorflow.Keras.Utils; using Tensorflow.Common.Types; +using System.Runtime.CompilerServices; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; namespace Tensorflow.Keras.Layers.Rnn @@ -30,7 +31,19 @@ namespace Tensorflow.Keras.Layers.Rnn private int _num_constants; protected IVariableV1 _kernel; protected IVariableV1 _bias; - protected IRnnCell _cell; + private IRnnCell _cell; + protected IRnnCell Cell + { + get + { + return _cell; + } + init + { + _cell = value; + _self_tracked_trackables.Add(_cell); + } + } public RNN(RNNArgs args) : base(PreConstruct(args)) { @@ -40,14 +53,14 @@ namespace Tensorflow.Keras.Layers.Rnn // if is StackedRnncell if (args.Cells != null) { - _cell = new StackedRNNCells(new StackedRNNCellsArgs + Cell = new StackedRNNCells(new StackedRNNCellsArgs { Cells = args.Cells }); } else { - _cell = args.Cell; + Cell = args.Cell; } // get input_shape @@ -65,7 +78,7 @@ namespace Tensorflow.Keras.Layers.Rnn if (_states == null) { // CHECK(Rinne): check if this is correct. - var nested = _cell.StateSize.MapStructure(x => null); + var nested = Cell.StateSize.MapStructure(x => null); _states = nested.AsNest().ToTensors(); } return _states; @@ -83,7 +96,7 @@ namespace Tensorflow.Keras.Layers.Rnn } // state_size is a array of ints or a positive integer - var state_size = _cell.StateSize.ToSingleShape(); + var state_size = Cell.StateSize.ToSingleShape(); // TODO(wanglongzhi2001),flat_output_size应该是什么类型的，Shape还是Tensor Func _get_output_shape; @@ -110,12 +123,12 @@ namespace Tensorflow.Keras.Layers.Rnn return output_shape; }; - Type type = _cell.GetType(); + Type type = Cell.GetType(); PropertyInfo output_size_info = type.GetProperty("output_size"); Shape output_shape; if (output_size_info != null) { - output_shape = nest.map_structure(_get_output_shape, _cell.OutputSize.ToSingleShape()); + output_shape = nest.map_structure(_get_output_shape, Cell.OutputSize.ToSingleShape()); // TODO(wanglongzhi2001),output_shape应该简单的就是一个元组还是一个Shape类型 output_shape = (output_shape.Length == 1 ? (int)output_shape[0] : output_shape); } @@ -171,7 +184,9 @@ namespace Tensorflow.Keras.Layers.Rnn public override void build(KerasShapesWrapper input_shape) { - object get_input_spec(Shape shape) + input_shape = new KerasShapesWrapper(input_shape.Shapes[0]); + + InputSpec get_input_spec(Shape shape) { var input_spec_shape = shape.as_int_list(); @@ -213,10 +228,13 @@ namespace Tensorflow.Keras.Layers.Rnn // numpy inputs. - if (!_cell.Built) + if (Cell is Layer layer && !layer.Built) { - _cell.build(input_shape); + layer.build(input_shape); + layer.Built = true; } + + this.built = true; } /// @@ -247,10 +265,10 @@ namespace Tensorflow.Keras.Layers.Rnn (inputs, initial_state, constants) = _process_inputs(inputs, initial_state, constants); - _maybe_reset_cell_dropout_mask(_cell); - if (_cell is StackedRNNCells) + _maybe_reset_cell_dropout_mask(Cell); + if (Cell is StackedRNNCells) { - var stack_cell = _cell as StackedRNNCells; + var stack_cell = Cell as StackedRNNCells; foreach (IRnnCell cell in stack_cell.Cells) { _maybe_reset_cell_dropout_mask(cell); @@ -300,10 +318,10 @@ namespace Tensorflow.Keras.Layers.Rnn bool is_tf_rnn_cell = false; if (constants is not null) { - if (!_cell.SupportOptionalArgs) + if (!Cell.SupportOptionalArgs) { throw new ValueError( - $"RNN cell {_cell} does not support constants." + + $"RNN cell {Cell} does not support constants." + $"Received: constants={constants}"); } @@ -312,7 +330,7 @@ namespace Tensorflow.Keras.Layers.Rnn constants = new Tensors(states.TakeLast(_num_constants).ToArray()); states = new Tensors(states.SkipLast(_num_constants).ToArray()); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; - var (output, new_states) = _cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); + var (output, new_states) = Cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); return (output, new_states.Single); }; } @@ -321,7 +339,7 @@ namespace Tensorflow.Keras.Layers.Rnn step = (inputs, states) => { states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states.First()) : states; - var (output, new_states) = _cell.Apply(inputs, states); + var (output, new_states) = Cell.Apply(inputs, states); return (output, new_states); }; } @@ -562,7 +580,7 @@ namespace Tensorflow.Keras.Layers.Rnn var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; var dtype = input.dtype; - Tensors init_state = _cell.GetInitialState(null, batch_size, dtype); + Tensors init_state = Cell.GetInitialState(null, batch_size, dtype); return init_state; } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index 22d0e277..551c20cd 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -32,19 +32,5 @@ namespace Tensorflow.Keras.Layers.Rnn }); return args; } - - public override void build(KerasShapesWrapper input_shape) - { - var single_shape = input_shape.ToSingleShape(); - var input_dim = single_shape[-1]; - _buildInputShape = input_shape; - - _kernel = add_weight("kernel", (single_shape[-1], args.Units), - initializer: args.KernelInitializer - //regularizer = self.kernel_regularizer, - //constraint = self.kernel_constraint, - //caching_device = default_caching_device, - ); - } } } \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 28a16ad4..fcb9ad1d 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -77,6 +77,11 @@ namespace Tensorflow.Keras.UnitTest.Layers var output = keras.layers.Dense(10).Apply(x); var model = keras.Model(inputs, output); model.summary(); + + model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy()); + var datax = np.ones((16, 10, 8), dtype: dtypes.float32); + var datay = np.ones((16)); + model.fit(datax, datay, epochs: 20); } [TestMethod] public void RNNForSimpleRNNCell() From 5bfe0982e93cbc3ee3e7e1afbbd66e3d445f5bdd Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 16 Jun 2023 16:15:27 +0800 Subject: [PATCH 09/77] feat: add exception catch to code generator. --- tools/Tensorflow.CodeGen/FunctionGenerator.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/Tensorflow.CodeGen/FunctionGenerator.cs b/tools/Tensorflow.CodeGen/FunctionGenerator.cs index 186e6a27..bb07dddf 100644 --- a/tools/Tensorflow.CodeGen/FunctionGenerator.cs +++ b/tools/Tensorflow.CodeGen/FunctionGenerator.cs @@ -83,6 +83,10 @@ namespace Tensorflow.CodeGen sb.AppendLine("}"); // try + sb.Append("catch(NotOkStatusException ex)\n{\n"); + sb.AppendLine("throw ex;"); + sb.AppendLine("}"); // catch + sb.Append("catch(Exception)\n{\n"); sb.AppendLine("}"); // catch From df7d700fb162ebe85ff1ae4ca831c7f9e9b1204a Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Wed, 14 Jun 2023 20:34:25 +0800 Subject: [PATCH 10/77] Add new feature: add LSTMCell and test --- .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 32 ++- .../ArgsDefinition/Rnn/SimpleRNNCellArgs.cs | 4 +- .../Keras/Layers/ILayersApi.cs | 12 + .../Operations/Initializers/Orthogonal.cs | 3 +- .../Operations/array_ops.cs | 43 ++++ src/TensorFlowNET.Keras/Layers/LayersApi.cs | 28 +++ .../Layers/Rnn/DropoutRNNCellMixin.cs | 4 +- .../Layers/Rnn/LSTMCell.cs | 232 +++++++++++++++++- .../Layers/Rnn/SimpleRNNCell.cs | 4 +- .../Layers/Rnn.Test.cs | 50 ++-- 10 files changed, 376 insertions(+), 36 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 594c99bb..786236e4 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -1,7 +1,35 @@ -namespace Tensorflow.Keras.ArgsDefinition.Rnn +using Newtonsoft.Json; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn { // TODO: complete the implementation - public class LSTMCellArgs : LayerArgs + public class LSTMCellArgs : AutoSerializeLayerArgs { + [JsonProperty("units")] + public int Units { get; set; } + // TODO(Rinne): lack of initialized value of Activation. Merging keras + // into tf.net could resolve it. + [JsonProperty("activation")] + public Activation Activation { get; set; } + [JsonProperty("recurrent_activation")] + public Activation RecurrentActivation { get; set; } + [JsonProperty("use_bias")] + public bool UseBias { get; set; } = true; + [JsonProperty("dropout")] + public float Dropout { get; set; } = .0f; + [JsonProperty("recurrent_dropout")] + public float RecurrentDropout { get; set; } = .0f; + [JsonProperty("kernel_initializer")] + public IInitializer KernelInitializer { get; set; } + [JsonProperty("recurrent_initializer")] + public IInitializer RecurrentInitializer { get; set; } + [JsonProperty("bias_initializer")] + public IInitializer BiasInitializer { get; set; } + [JsonProperty("unit_forget_bias")] + public bool UnitForgetBias { get; set; } = true; + [JsonProperty("implementation")] + public int Implementation { get; set; } = 2; + } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs index 1dfcbe9c..d21d6190 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs @@ -1,7 +1,4 @@ using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.Text; namespace Tensorflow.Keras.ArgsDefinition.Rnn { @@ -25,5 +22,6 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public IInitializer RecurrentInitializer { get; set; } [JsonProperty("bias_initializer")] public IInitializer BiasInitializer { get; set; } + } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 3b223816..a19508d4 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -160,6 +160,18 @@ namespace Tensorflow.Keras.Layers public ILayer Normalization(Shape? input_shape = null, int? axis = -1, float? mean = null, float? variance = null, bool invert = false); public ILayer LeakyReLU(float alpha = 0.3f); + public IRnnCell LSTMCell(int uints, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + bool unit_forget_bias = true, + float dropout = 0f, + float recurrent_dropout = 0f, + int implementation = 2); + public ILayer LSTM(int units, Activation activation = null, Activation recurrent_activation = null, diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs index 88673bb5..ae873374 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs @@ -58,8 +58,7 @@ public class Orthogonal : IInitializer if (num_rows < num_cols) { - // q = tf.linalg.matrix_transpose(q); - throw new NotImplementedException(""); + q = array_ops.matrix_transpose(q); } return _gain * tf.reshape(q, shape); diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index ca9e5fae..c4ec974b 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -971,6 +971,49 @@ namespace Tensorflow }); } + /// + /// Transposes last two dimensions of tensor `a`. + /// For example: + /// python + /// x = tf.constant([[1, 2, 3], [4, 5, 6]]) + /// tf.matrix_transpose(x) # [[1, 4], + /// # [2, 5], + /// # [3, 6]] + /// + /// Matrix with two batch dimensions. + /// x.shape is [1, 2, 3, 4] + /// tf.linalg.matrix_transpose(x) is shape [1, 2, 4, 3] + /// + /// + /// + /// + /// + /// + public static Tensor matrix_transpose(Tensor a, string name = "matrix_transpose", bool conjugate = false) + { + return tf_with(ops.name_scope(name, "transpose", new { a }), scope => + { + var a_shape = a.shape; + var ndims = a.shape.ndim; + Axis perm; + if(ndims != 0) + { + if (ndims < 2) + { + throw new ValueError("Argument `a` should be a (batch) matrix with rank " + + $">= 2. Received `a` = {a} with shape: {a_shape}"); + } + perm = new Axis(Enumerable.Range(0, ndims - 2).Concat(new int[] { ndims - 1, ndims - 2 }).ToArray()); + } + else + { + var a_rank = a.rank; + perm = new Axis(Enumerable.Range(0, a_rank - 2).Concat(new int[] { a_rank - 1, a_rank - 2 }).ToArray()); + } + return transpose(a, perm:perm, conjugate:conjugate); + }); + } + public static Tensor[] split(Tensor value, Tensor size_splits, int axis, int num = -1, string name = "split") { diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index dd25122d..66c3cdc1 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -702,6 +702,7 @@ namespace Tensorflow.Keras.Layers UseBias = use_bias, KernelInitializer = GetInitializerByName(kernel_initializer), RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), Dropout = dropout, RecurrentDropout = recurrent_dropout }); @@ -786,6 +787,33 @@ namespace Tensorflow.Keras.Layers TimeMajor = time_major }); + + public IRnnCell LSTMCell(int uints, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", // TODO(Wanglongzhi2001),glorot_uniform has not been developed. + string bias_initializer = "zeros", + bool unit_forget_bias = true, + float dropout = 0f, + float recurrent_dropout = 0f, + int implementation = 2) + => new LSTMCell(new LSTMCellArgs + { + Units = uints, + Activation = keras.activations.GetActivationFromName(activation), + RecurrentActivation = keras.activations.GetActivationFromName(recurrent_activation), + UseBias = use_bias, + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + UnitForgetBias = unit_forget_bias, + Dropout = dropout, + RecurrentDropout = recurrent_dropout, + Implementation = implementation + }); + /// /// Long Short-Term Memory layer - Hochreiter 1997. /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index d2669ccc..1cc36d34 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -41,7 +41,7 @@ namespace Tensorflow.Keras.Layers.Rnn } - public Tensors? get_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + public Tensors? get_dropout_mask_for_cell(Tensors input, bool training, int count = 1) { if (dropout == 0f) return null; @@ -53,7 +53,7 @@ namespace Tensorflow.Keras.Layers.Rnn } // Get the recurrent dropout mask for RNN cell. - public Tensors? get_recurrent_dropout_maskcell_for_cell(Tensors input, bool training, int count = 1) + public Tensors? get_recurrent_dropout_mask_for_cell(Tensors input, bool training, int count = 1) { if (dropout == 0f) return null; diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index a622c91a..94d98e13 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -1,16 +1,240 @@ -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Serilog.Core; +using System.Diagnostics; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Layers.Rnn { - public class LSTMCell : Layer + /// + /// Cell class for the LSTM layer. + /// See [the Keras RNN API guide](https://www.tensorflow.org/guide/keras/rnn) + /// for details about the usage of RNN API. + /// This class processes one step within the whole time sequence input, whereas + /// `tf.keras.layer.LSTM` processes the whole sequence. + /// + public class LSTMCell : DropoutRNNCellMixin { - LSTMCellArgs args; + LSTMCellArgs _args; + IVariableV1 _kernel; + IVariableV1 _recurrent_kernel; + IInitializer _bias_initializer; + IVariableV1 _bias; + GeneralizedTensorShape _state_size; + GeneralizedTensorShape _output_size; + public override GeneralizedTensorShape StateSize => _state_size; + public override GeneralizedTensorShape OutputSize => _output_size; + + public override bool IsTFRnnCell => true; + + public override bool SupportOptionalArgs => false; public LSTMCell(LSTMCellArgs args) : base(args) { - this.args = args; + _args = args; + if (args.Units <= 0) + { + throw new ValueError( + $"units must be a positive integer, got {args.Units}"); + } + _args.Dropout = Math.Min(1f, Math.Max(0f, this._args.Dropout)); + _args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this._args.RecurrentDropout)); + if (_args.RecurrentDropout != 0f && _args.Implementation != 1) + { + Debug.WriteLine("RNN `implementation=2` is not supported when `recurrent_dropout` is set." + + "Using `implementation=1`."); + _args.Implementation = 1; + } + + _state_size = new GeneralizedTensorShape(_args.Units, 2); + _output_size = new GeneralizedTensorShape(_args.Units); + + + } + + public override void build(KerasShapesWrapper input_shape) + { + var single_shape = input_shape.ToSingleShape(); + var input_dim = single_shape[-1]; + _kernel = add_weight("kernel", (input_dim, _args.Units * 4), + initializer: _args.KernelInitializer + ); + + _recurrent_kernel = add_weight("recurrent_kernel", (_args.Units, _args.Units * 4), + initializer: _args.RecurrentInitializer + ); + + if (_args.UseBias) + { + if (_args.UnitForgetBias) + { + Tensor bias_initializer() + { + return keras.backend.concatenate( + new Tensors( + _args.BiasInitializer.Apply(new InitializerArgs(shape: (_args.Units))), + tf.ones_initializer.Apply(new InitializerArgs(shape: (_args.Units))), + _args.BiasInitializer.Apply(new InitializerArgs(shape: (_args.Units)))), axis: 0); + } + } + else + { + _bias_initializer = _args.BiasInitializer; + } + _bias = add_weight("bias", (_args.Units * 4), + initializer: _args.BiasInitializer); + } + built = true; + } + protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) + { + var h_tm1 = states[0]; // previous memory state + var c_tm1 = states[1]; // previous carry state + + var dp_mask = get_dropout_mask_for_cell(inputs, training.Value, count: 4); + var rec_dp_mask = get_recurrent_dropout_mask_for_cell( + h_tm1, training.Value, count: 4); + + + Tensor c; + Tensor o; + if (_args.Implementation == 1) + { + Tensor inputs_i; + Tensor inputs_f; + Tensor inputs_c; + Tensor inputs_o; + if (0f < _args.Dropout && _args.Dropout < 1f) + { + inputs_i = inputs * dp_mask[0]; + inputs_f = inputs * dp_mask[1]; + inputs_c = inputs * dp_mask[2]; + inputs_o = inputs * dp_mask[3]; + } + else + { + inputs_i = inputs; + inputs_f = inputs; + inputs_c = inputs; + inputs_o = inputs; + } + var k = tf.split(_kernel.AsTensor(), num_split: 4, axis: 1); + Tensor k_i = k[0], k_f = k[1], k_c = k[2], k_o = k[3]; + var x_i = math_ops.matmul(inputs_i, k_i); + var x_f = math_ops.matmul(inputs_f, k_f); + var x_c = math_ops.matmul(inputs_c, k_c); + var x_o = math_ops.matmul(inputs_o, k_o); + if(_args.UseBias) + { + var b = tf.split(_bias.AsTensor(), num_split: 4, axis: 0); + Tensor b_i = b[0], b_f = b[1], b_c = b[2], b_o = b[3]; + x_i = gen_nn_ops.bias_add(x_i, b_i); + x_f = gen_nn_ops.bias_add(x_f, b_f); + x_c = gen_nn_ops.bias_add(x_c, b_c); + x_o = gen_nn_ops.bias_add(x_o, b_o); + } + + Tensor h_tm1_i; + Tensor h_tm1_f; + Tensor h_tm1_c; + Tensor h_tm1_o; + if (0f < _args.RecurrentDropout && _args.RecurrentDropout < 1f) + { + h_tm1_i = h_tm1 * rec_dp_mask[0]; + h_tm1_f = h_tm1 * rec_dp_mask[1]; + h_tm1_c = h_tm1 * rec_dp_mask[2]; + h_tm1_o = h_tm1 * rec_dp_mask[3]; + } + else + { + h_tm1_i = h_tm1; + h_tm1_f = h_tm1; + h_tm1_c = h_tm1; + h_tm1_o = h_tm1; + } + var x = new Tensor[] { x_i, x_f, x_c, x_o }; + var h_tm1_array = new Tensor[] { h_tm1_i, h_tm1_f, h_tm1_c, h_tm1_o }; + (c, o) = _compute_carry_and_output(x, h_tm1_array, c_tm1); + } + else + { + if (0f < _args.Dropout && _args.Dropout < 1f) + inputs = inputs * dp_mask[0]; + var z = math_ops.matmul(inputs, _kernel.AsTensor()); + z += math_ops.matmul(h_tm1, _recurrent_kernel.AsTensor()); + if (_args.UseBias) + { + z = tf.nn.bias_add(z, _bias); + } + var z_array = tf.split(z, num_split: 4, axis: 1); + (c, o) = _compute_carry_and_output_fused(z_array, c_tm1); + } + var h = o * _args.Activation.Apply(c); + // 这里是因为 Tensors 类初始化的时候会把第一个元素之后的元素打包成一个数组 + return new Tensors(h, h, c); + } + + /// + /// Computes carry and output using split kernels. + /// + /// + /// + /// + /// + /// + public Tensors _compute_carry_and_output(Tensor[] x, Tensor[] h_tm1, Tensor c_tm1) + { + Tensor x_i = x[0], x_f = x[1], x_c = x[2], x_o = x[3]; + Tensor h_tm1_i = h_tm1[0], h_tm1_f = h_tm1[1], h_tm1_c = h_tm1[2], + h_tm1_o = h_tm1[3]; + + var _recurrent_kernel_tensor = _recurrent_kernel.AsTensor(); + var startIndex = _recurrent_kernel_tensor.shape[0]; + var endIndex = _recurrent_kernel_tensor.shape[1]; + var _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, + new[] { 0, 0 }, new[] { startIndex, _args.Units }); + var i = _args.RecurrentActivation.Apply( + x_i + math_ops.matmul(h_tm1_i, _recurrent_kernel_slice)); + _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, + new[] { 0, _args.Units }, new[] { startIndex, _args.Units * 2}); + var f = _args.RecurrentActivation.Apply( + x_f + math_ops.matmul(h_tm1_f, _recurrent_kernel_slice)); + _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, + new[] { 0, _args.Units * 2 }, new[] { startIndex, _args.Units * 3 }); + var c = f * c_tm1 + i * _args.Activation.Apply( + x_c + math_ops.matmul(h_tm1_c, _recurrent_kernel_slice)); + _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, + new[] { 0, _args.Units * 3 }, new[] { startIndex, endIndex }); + var o = _args.RecurrentActivation.Apply( + x_o + math_ops.matmul(h_tm1_o, _recurrent_kernel_slice)); + + return new Tensors(c, o); + } + + /// + /// Computes carry and output using fused kernels. + /// + /// + /// + /// + public Tensors _compute_carry_and_output_fused(Tensor[] z, Tensor c_tm1) + { + Tensor z0 = z[0], z1 = z[1], z2 = z[2], z3 = z[3]; + var i = _args.RecurrentActivation.Apply(z0); + var f = _args.RecurrentActivation.Apply(z1); + var c = f * c_tm1 + i * _args.RecurrentActivation.Apply(z2); + var o = _args.RecurrentActivation.Apply(z3); + return new Tensors(c, o); + } + + public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) + { + return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size.Value, dtype.Value); } } + + } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 3b4b9419..d318dc45 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -74,8 +74,8 @@ namespace Tensorflow.Keras.Layers.Rnn { // TODO(Rinne): check if it will have multiple tensors when not nested. Tensors prev_output = Nest.IsNested(states) ? new Tensors(states[0]) : states; - var dp_mask = get_dropout_maskcell_for_cell(inputs, training.Value); - var rec_dp_mask = get_recurrent_dropout_maskcell_for_cell(prev_output, training.Value); + var dp_mask = get_dropout_mask_for_cell(inputs, training.Value); + var rec_dp_mask = get_recurrent_dropout_mask_for_cell(prev_output, training.Value); Tensor h; var ranks = inputs.rank; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index fcb9ad1d..54ea1565 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -21,21 +21,6 @@ namespace Tensorflow.Keras.UnitTest.Layers [TestMethod] public void SimpleRNNCell() { - //var cell = tf.keras.layers.SimpleRNNCell(64, dropout: 0.5f, recurrent_dropout: 0.5f); - //var h0 = new Tensors { tf.zeros(new Shape(4, 64)) }; - //var x = tf.random.normal((4, 100)); - //var (y, h1) = cell.Apply(inputs: x, states: h0); - //var h2 = h1; - //Assert.AreEqual((4, 64), y.shape); - //Assert.AreEqual((4, 64), h2[0].shape); - - //var model = keras.Sequential(new List - //{ - // keras.layers.InputLayer(input_shape: (4,100)), - // keras.layers.SimpleRNNCell(64) - //}); - //model.summary(); - var cell = tf.keras.layers.SimpleRNNCell(64, dropout: 0.5f, recurrent_dropout: 0.5f); var h0 = new Tensors { tf.zeros(new Shape(4, 64)) }; var x = tf.random.normal((4, 100)); @@ -59,6 +44,17 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual((32, 4), state[0].shape); } + [TestMethod] + public void LSTMCell() + { + var inputs = tf.ones((2, 100)); + var states = new Tensors { tf.zeros((2, 4)), tf.zeros((2, 4)) }; + var rnn = tf.keras.layers.LSTMCell(4); + var (output, new_states) = rnn.Apply(inputs, states); + Assert.AreEqual((2, 4), output.shape); + Assert.AreEqual((2, 4), new_states[0].shape); + } + [TestMethod] public void SimpleRNN() { @@ -105,15 +101,27 @@ namespace Tensorflow.Keras.UnitTest.Layers } [TestMethod] - public void WlzTest() + public void RNNForLSTMCell() { - long[] b = { 1, 2, 3 }; - - Shape a = new Shape(Unknown).concatenate(b); - Console.WriteLine(a); - + var inputs = tf.ones((5, 10, 8)); + var rnn = tf.keras.layers.RNN(tf.keras.layers.LSTMCell(4)); + var output = rnn.Apply(inputs); + Console.WriteLine($"output: {output}"); + Assert.AreEqual((5, 4), output.shape); } + [TestMethod] + public void MyTest() + { + var a = tf.zeros((2, 3)); + var b = tf.ones_like(a); + var c = tf.ones((3,4)); + + var d = new Tensors { a, b, c }; + var (A, BC) = d; + Console.WriteLine($"A:{A}"); + Console.WriteLine($"BC:{BC}"); + } } } From 6b30902ee88c7ce608bf7a938eac3dcc1664546b Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 16 Jun 2023 18:55:23 +0800 Subject: [PATCH 11/77] fix: error after merging LSTM support. --- .../Common/Types/GeneralizedTensorShape.cs | 15 ------- .../Common/Types/NestList.cs | 7 ++- .../Keras/Layers/Rnn/IRnnCell.cs | 4 +- src/TensorFlowNET.Core/Numpy/Shape.cs | 24 +++++++++- .../Operations/NnOps/RNNCell.cs | 4 +- .../Layers/Rnn/DropoutRNNCellMixin.cs | 4 +- .../Layers/Rnn/LSTMCell.cs | 21 +++------ src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 44 +++++++------------ .../Layers/Rnn/SimpleRNNCell.cs | 12 ++--- .../Layers/Rnn/StackedRNNCells.cs | 20 ++++----- src/TensorFlowNET.Keras/Utils/RnnUtils.cs | 13 +++--- 11 files changed, 79 insertions(+), 89 deletions(-) diff --git a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs index 40190315..986136f4 100644 --- a/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs +++ b/src/TensorFlowNET.Core/Common/Types/GeneralizedTensorShape.cs @@ -7,21 +7,6 @@ namespace Tensorflow.Common.Types { public class GeneralizedTensorShape: Nest { - ////public TensorShapeConfig[] Shapes { get; set; } - ///// - ///// create a single-dim generalized Tensor shape. - ///// - ///// - //public GeneralizedTensorShape(int dim, int size = 1) - //{ - // var elem = new TensorShapeConfig() { Items = new long?[] { dim } }; - // Shapes = Enumerable.Repeat(elem, size).ToArray(); - // //Shapes = new TensorShapeConfig[size]; - // //Shapes.Initialize(new TensorShapeConfig() { Items = new long?[] { dim } }); - // //Array.Initialize(Shapes, new TensorShapeConfig() { Items = new long?[] { dim } }); - // ////Shapes = new TensorShapeConfig[] { new TensorShapeConfig() { Items = new long?[] { dim } } }; - //} - public GeneralizedTensorShape(Shape value, string? name = null) { NodeValue = value; diff --git a/src/TensorFlowNET.Core/Common/Types/NestList.cs b/src/TensorFlowNET.Core/Common/Types/NestList.cs index e38675da..1e0d272b 100644 --- a/src/TensorFlowNET.Core/Common/Types/NestList.cs +++ b/src/TensorFlowNET.Core/Common/Types/NestList.cs @@ -15,7 +15,12 @@ namespace Tensorflow.Common.Types public int ShallowNestedCount => Values.Count; public int TotalNestedCount => Values.Count; - + + public NestList(params T[] values) + { + Values = new List(values); + } + public NestList(IEnumerable values) { Values = new List(values); diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs index 8614391a..8d6fbc97 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -10,11 +10,11 @@ namespace Tensorflow.Keras.Layers.Rnn /// /// If the derived class tends to not implement it, please return null. /// - GeneralizedTensorShape? StateSize { get; } + INestStructure? StateSize { get; } /// /// If the derived class tends to not implement it, please return null. /// - GeneralizedTensorShape? OutputSize { get; } + INestStructure? OutputSize { get; } /// /// Whether the optional RNN args are supported when appying the layer. /// In other words, whether `Apply` is overwrited with process of `RnnOptionalArgs`. diff --git a/src/TensorFlowNET.Core/Numpy/Shape.cs b/src/TensorFlowNET.Core/Numpy/Shape.cs index c339f12d..cbbf66b4 100644 --- a/src/TensorFlowNET.Core/Numpy/Shape.cs +++ b/src/TensorFlowNET.Core/Numpy/Shape.cs @@ -19,13 +19,14 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Tensorflow.Common.Types; using Tensorflow.Keras.Saving.Common; using Tensorflow.NumPy; namespace Tensorflow { [JsonConverter(typeof(CustomizedShapeJsonConverter))] - public class Shape + public class Shape : INestStructure { public int ndim => _dims == null ? -1 : _dims.Length; long[] _dims; @@ -41,6 +42,27 @@ namespace Tensorflow } } + public NestType NestType => NestType.List; + + public int ShallowNestedCount => ndim; + /// + /// The total item count of depth 1 of the nested structure. + /// For example, [1, 2, [3, 4, 5]] has TotalNestedCount = 5. + /// + public int TotalNestedCount => ndim; + + public IEnumerable Flatten() => dims.Select(x => x); + + public INestStructure MapStructure(Func func) + { + return new NestList(dims.Select(x => func(x))); + } + + public Nest AsNest() + { + return new NestList(Flatten()).AsNest(); + } + #region https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/proposals/csharp-8.0/ranges public int Length => ndim; public long[] Slice(int start, int length) diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index b651089a..e488c47e 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -185,8 +185,8 @@ namespace Tensorflow { throw new NotImplementedException(); } - public GeneralizedTensorShape StateSize => throw new NotImplementedException(); - public GeneralizedTensorShape OutputSize => throw new NotImplementedException(); + public INestStructure StateSize => throw new NotImplementedException(); + public INestStructure OutputSize => throw new NotImplementedException(); public bool IsTFRnnCell => throw new NotImplementedException(); public bool SupportOptionalArgs => throw new NotImplementedException(); } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 1cc36d34..75feb8ea 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -18,8 +18,8 @@ namespace Tensorflow.Keras.Layers.Rnn } - public abstract GeneralizedTensorShape StateSize { get; } - public abstract GeneralizedTensorShape OutputSize { get; } + public abstract INestStructure StateSize { get; } + public abstract INestStructure OutputSize { get; } public abstract bool SupportOptionalArgs { get; } public virtual Tensors GetInitialState(Tensors inputs, Tensor batch_size, TF_DataType dtype) { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 94d98e13..17042767 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -22,13 +22,11 @@ namespace Tensorflow.Keras.Layers.Rnn IVariableV1 _recurrent_kernel; IInitializer _bias_initializer; IVariableV1 _bias; - GeneralizedTensorShape _state_size; - GeneralizedTensorShape _output_size; - public override GeneralizedTensorShape StateSize => _state_size; + INestStructure _state_size; + INestStructure _output_size; + public override INestStructure StateSize => _state_size; - public override GeneralizedTensorShape OutputSize => _output_size; - - public override bool IsTFRnnCell => true; + public override INestStructure OutputSize => _output_size; public override bool SupportOptionalArgs => false; public LSTMCell(LSTMCellArgs args) @@ -49,10 +47,8 @@ namespace Tensorflow.Keras.Layers.Rnn _args.Implementation = 1; } - _state_size = new GeneralizedTensorShape(_args.Units, 2); - _output_size = new GeneralizedTensorShape(_args.Units); - - + _state_size = new NestList(_args.Units, _args.Units); + _output_size = new NestNode(_args.Units); } public override void build(KerasShapesWrapper input_shape) @@ -229,11 +225,6 @@ namespace Tensorflow.Keras.Layers.Rnn var o = _args.RecurrentActivation.Apply(z3); return new Tensors(c, o); } - - public Tensors get_initial_state(Tensors inputs = null, long? batch_size = null, TF_DataType? dtype = null) - { - return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size.Value, dtype.Value); - } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index f99bc23a..0aeacc25 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -86,7 +86,7 @@ namespace Tensorflow.Keras.Layers.Rnn set { _states = value; } } - private OneOf> compute_output_shape(Shape input_shape) + private INestStructure compute_output_shape(Shape input_shape) { var batch = input_shape[0]; var time_step = input_shape[1]; @@ -96,13 +96,15 @@ namespace Tensorflow.Keras.Layers.Rnn } // state_size is a array of ints or a positive integer - var state_size = Cell.StateSize.ToSingleShape(); + var state_size = Cell.StateSize; + if(state_size?.TotalNestedCount == 1) + { + state_size = new NestList(state_size.Flatten().First()); + } - // TODO(wanglongzhi2001),flat_output_size应该是什么类型的，Shape还是Tensor - Func _get_output_shape; - _get_output_shape = (flat_output_size) => + Func _get_output_shape = (flat_output_size) => { - var output_dim = flat_output_size.as_int_list(); + var output_dim = new Shape(flat_output_size).as_int_list(); Shape output_shape; if (_args.ReturnSequences) { @@ -125,31 +127,28 @@ namespace Tensorflow.Keras.Layers.Rnn Type type = Cell.GetType(); PropertyInfo output_size_info = type.GetProperty("output_size"); - Shape output_shape; + INestStructure output_shape; if (output_size_info != null) { - output_shape = nest.map_structure(_get_output_shape, Cell.OutputSize.ToSingleShape()); - // TODO(wanglongzhi2001),output_shape应该简单的就是一个元组还是一个Shape类型 - output_shape = (output_shape.Length == 1 ? (int)output_shape[0] : output_shape); + output_shape = Nest.MapStructure(_get_output_shape, Cell.OutputSize); } else { - output_shape = _get_output_shape(state_size); + output_shape = new NestNode(_get_output_shape(state_size.Flatten().First())); } if (_args.ReturnState) { - Func _get_state_shape; - _get_state_shape = (flat_state) => + Func _get_state_shape = (flat_state) => { - var state_shape = new int[] { (int)batch }.concat(flat_state.as_int_list()); + var state_shape = new int[] { (int)batch }.concat(new Shape(flat_state).as_int_list()); return new Shape(state_shape); }; - var state_shape = _get_state_shape(state_size); + var state_shape = Nest.MapStructure(_get_state_shape, state_size); - return new List { output_shape, state_shape }; + return new Nest(new[] { output_shape, state_shape } ); } else { @@ -435,7 +434,7 @@ namespace Tensorflow.Keras.Layers.Rnn tmp.add(tf.math.count_nonzero(s.Single())); } var non_zero_count = tf.add_n(tmp); - //initial_state = tf.cond(non_zero_count > 0, () => States, () => initial_state); + initial_state = tf.cond(non_zero_count > 0, States, initial_state); if ((int)non_zero_count.numpy() > 0) { initial_state = States; @@ -445,16 +444,7 @@ namespace Tensorflow.Keras.Layers.Rnn { initial_state = States; } - // TODO(Wanglongzhi2001), -// initial_state = tf.nest.map_structure( -//# When the layer has a inferred dtype, use the dtype from the -//# cell. -// lambda v: tf.cast( -// v, self.compute_dtype or self.cell.compute_dtype -// ), -// initial_state, -// ) - + //initial_state = Nest.MapStructure(v => tf.cast(v, this.), initial_state); } else if (initial_state is null) { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index d318dc45..8fdc598e 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -24,11 +24,11 @@ namespace Tensorflow.Keras.Layers.Rnn IVariableV1 _kernel; IVariableV1 _recurrent_kernel; IVariableV1 _bias; - GeneralizedTensorShape _state_size; - GeneralizedTensorShape _output_size; + INestStructure _state_size; + INestStructure _output_size; - public override GeneralizedTensorShape StateSize => _state_size; - public override GeneralizedTensorShape OutputSize => _output_size; + public override INestStructure StateSize => _state_size; + public override INestStructure OutputSize => _output_size; public override bool SupportOptionalArgs => false; public SimpleRNNCell(SimpleRNNCellArgs args) : base(args) @@ -41,8 +41,8 @@ namespace Tensorflow.Keras.Layers.Rnn } this._args.Dropout = Math.Min(1f, Math.Max(0f, this._args.Dropout)); this._args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this._args.RecurrentDropout)); - _state_size = new GeneralizedTensorShape(args.Units); - _output_size = new GeneralizedTensorShape(args.Units); + _state_size = new NestNode(args.Units); + _output_size = new NestNode(args.Units); } public override void build(KerasShapesWrapper input_shape) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index fb74d6d2..3e7b227c 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -1,10 +1,8 @@ using System; -using System.Collections.Generic; using System.ComponentModel; using System.Linq; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; @@ -38,24 +36,24 @@ namespace Tensorflow.Keras.Layers.Rnn public bool SupportOptionalArgs => false; - public GeneralizedTensorShape StateSize + public INestStructure StateSize { get { if (_reverse_state_order) { var state_sizes = Cells.Reverse().Select(cell => cell.StateSize); - return new GeneralizedTensorShape(new Nest(state_sizes.Select(s => new Nest(s)))); + return new Nest(state_sizes); } else { var state_sizes = Cells.Select(cell => cell.StateSize); - return new GeneralizedTensorShape(new Nest(state_sizes.Select(s => new Nest(s)))); + return new Nest(state_sizes); } } } - public GeneralizedTensorShape OutputSize + public INestStructure OutputSize { get { @@ -66,7 +64,7 @@ namespace Tensorflow.Keras.Layers.Rnn } else if (RnnUtils.is_multiple_state(lastCell.StateSize)) { - return lastCell.StateSize.First(); + return new NestNode(lastCell.StateSize.Flatten().First()); } else { @@ -89,7 +87,7 @@ namespace Tensorflow.Keras.Layers.Rnn protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) { // Recover per-cell states. - var state_size = _reverse_state_order ? new GeneralizedTensorShape(StateSize.Reverse()) : StateSize; + var state_size = _reverse_state_order ? new NestList(StateSize.Flatten().Reverse()) : StateSize; var nested_states = Nest.PackSequenceAs(state_size, Nest.Flatten(states).ToArray()); var new_nest_states = Nest.Empty; @@ -118,20 +116,20 @@ namespace Tensorflow.Keras.Layers.Rnn layer.build(shape); layer.Built = true; } - GeneralizedTensorShape output_dim; + INestStructure output_dim; if(cell.OutputSize is not null) { output_dim = cell.OutputSize; } else if (RnnUtils.is_multiple_state(cell.StateSize)) { - output_dim = cell.StateSize.First(); + output_dim = new NestNode(cell.StateSize.Flatten().First()); } else { output_dim = cell.StateSize; } - shape = new Shape(new long[] { shape.dims[0] }.Concat(output_dim.ToSingleShape().dims).ToArray()); + shape = new Shape(new long[] { shape.dims[0] }.Concat(output_dim.Flatten()).ToArray()); } this.Built = true; } diff --git a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs index 7ff3f9fb..e8700c1f 100644 --- a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs +++ b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs @@ -10,12 +10,11 @@ namespace Tensorflow.Keras.Utils { internal static class RnnUtils { - internal static Tensors generate_zero_filled_state(Tensor batch_size_tensor, GeneralizedTensorShape state_size, TF_DataType dtype) + internal static Tensors generate_zero_filled_state(Tensor batch_size_tensor, INestStructure state_size, TF_DataType dtype) { - Func create_zeros; - create_zeros = (GeneralizedTensorShape unnested_state_size) => + Func create_zeros = (unnested_state_size) => { - var flat_dims = unnested_state_size.ToSingleShape().dims; + var flat_dims = new Shape(unnested_state_size).dims; var init_state_size = new Tensor[] { batch_size_tensor }. Concat(flat_dims.Select(x => tf.constant(x, dtypes.int32))).ToArray(); return array_ops.zeros(init_state_size, dtype: dtype); @@ -24,11 +23,11 @@ namespace Tensorflow.Keras.Utils // TODO(Rinne): map structure with nested tensors. if(state_size.TotalNestedCount > 1) { - return new Tensors(state_size.Flatten().Select(s => create_zeros(new GeneralizedTensorShape(s))).ToArray()); + return new Tensors(state_size.Flatten().Select(s => create_zeros(s)).ToArray()); } else { - return create_zeros(state_size); + return create_zeros(state_size.Flatten().First()); } } @@ -96,7 +95,7 @@ namespace Tensorflow.Keras.Utils /// /// /// - public static bool is_multiple_state(GeneralizedTensorShape state_size) + public static bool is_multiple_state(INestStructure state_size) { return state_size.TotalNestedCount > 1; } From 0114885ed775a2ef9847b64c582039b8324c10d6 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 16 Jun 2023 19:06:58 +0800 Subject: [PATCH 12/77] feat: update some gen_ops. --- .../Operations/gen_array_ops.cs | 499 +++++++++- .../Operations/gen_functional_ops.cs | 57 ++ .../Operations/gen_io_ops.cs | 936 ++++++++++++++++-- .../Operations/gen_list_ops.cs | 81 ++ .../Operations/gen_math_ops.cs | 588 ++++++++++- .../Operations/gen_nn_ops.cs | 409 ++++++++ tools/Tensorflow.CodeGen/GenOpsWriter.cs | 1 + 7 files changed, 2450 insertions(+), 121 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs index 9810d32f..8367c2f9 100644 --- a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs @@ -2,6 +2,7 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; @@ -25,6 +26,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatrixBandPart", name) { args = new object[] { input, num_lower, num_upper }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -76,6 +81,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatrixDiag", name) { args = new object[] { diagonal }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -125,6 +134,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatrixDiagPart", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -175,6 +188,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatrixSetDiag", name) { args = new object[] { input, diagonal }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -238,6 +255,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchToSpace", name) { args = new object[] { input, crops }, attrs = new Dictionary() { ["block_size"] = block_size } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -301,6 +322,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchToSpaceND", name) { args = new object[] { input, block_shape, crops }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -407,6 +432,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Bitcast", name) { args = new object[] { input }, attrs = new Dictionary() { ["type"] = type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -464,6 +493,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BroadcastArgs", name) { args = new object[] { s0, s1 }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -520,6 +553,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BroadcastGradientArgs", name) { args = new object[] { s0, s1 }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -607,6 +644,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BroadcastTo", name) { args = new object[] { input, shape }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -689,6 +730,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CheckNumerics", name) { args = new object[] { tensor }, attrs = new Dictionary() { ["message"] = message } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -752,6 +797,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CheckNumericsV2", name) { args = new object[] { tensor }, attrs = new Dictionary() { ["message"] = message } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -803,6 +852,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Concat", name) { args = new object[] { concat_dim, values }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -871,6 +924,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ConcatOffset", name) { args = new object[] { concat_dim, shape }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -925,6 +982,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ConcatV2", name) { args = new object[] { values, axis }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -986,6 +1047,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ConjugateTranspose", name) { args = new object[] { x, perm }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1041,6 +1106,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Const", name) { args = new object[] { }, attrs = new Dictionary() { ["value"] = value, ["dtype"] = dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1098,6 +1167,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DebugGradientIdentity", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1182,6 +1255,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DeepCopy", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1330,6 +1407,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DepthToSpace", name) { args = new object[] { input }, attrs = new Dictionary() { ["block_size"] = block_size, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1452,6 +1533,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Dequantize", name) { args = new object[] { input, min_range, max_range }, attrs = new Dictionary() { ["mode"] = mode, ["narrow_range"] = narrow_range, ["axis"] = axis, ["dtype"] = dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1532,6 +1617,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Diag", name) { args = new object[] { diagonal }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1603,6 +1692,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DiagPart", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1674,6 +1767,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EditDistance", name) { args = new object[] { hypothesis_indices, hypothesis_values, hypothesis_shape, truth_indices, truth_values, truth_shape }, attrs = new Dictionary() { ["normalize"] = normalize } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1731,6 +1828,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Empty", name) { args = new object[] { shape }, attrs = new Dictionary() { ["dtype"] = dtype, ["init"] = init } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1793,6 +1894,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EnsureShape", name) { args = new object[] { input }, attrs = new Dictionary() { ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1878,6 +1983,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ExpandDims", name) { args = new object[] { input, dim }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1954,6 +2063,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ExtractImagePatches", name) { args = new object[] { images }, attrs = new Dictionary() { ["ksizes"] = ksizes, ["strides"] = strides, ["rates"] = rates, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2030,6 +2143,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ExtractVolumePatches", name) { args = new object[] { input }, attrs = new Dictionary() { ["ksizes"] = ksizes, ["strides"] = strides, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2110,6 +2227,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxArgs", name) { args = new object[] { inputs }, attrs = new Dictionary() { ["min"] = min, ["max"] = max, ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2168,6 +2289,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxArgsGradient", name) { args = new object[] { gradients, inputs }, attrs = new Dictionary() { ["min"] = min, ["max"] = max, ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2254,6 +2379,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxVars", name) { args = new object[] { inputs, min, max }, attrs = new Dictionary() { ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2320,6 +2449,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxVarsGradient", name) { args = new object[] { gradients, inputs, min, max }, attrs = new Dictionary() { ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2407,6 +2540,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxVarsPerChannel", name) { args = new object[] { inputs, min, max }, attrs = new Dictionary() { ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2473,6 +2610,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeQuantWithMinMaxVarsPerChannelGradient", name) { args = new object[] { gradients, inputs, min, max }, attrs = new Dictionary() { ["num_bits"] = num_bits, ["narrow_range"] = narrow_range } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2551,6 +2692,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Fill", name) { args = new object[] { dims, value }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2636,6 +2781,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Fingerprint", name) { args = new object[] { data, method }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2717,6 +2866,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Gather", name) { args = new object[] { params_, indices }, attrs = new Dictionary() { ["validate_indices"] = validate_indices } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2877,6 +3030,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "GatherNd", name) { args = new object[] { params_, indices }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2961,6 +3118,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "GatherV2", name) { args = new object[] { params_, indices, axis }, attrs = new Dictionary() { ["batch_dims"] = batch_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3023,6 +3184,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "GuaranteeConst", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3072,6 +3237,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Identity", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3129,24 +3298,27 @@ public static class gen_array_ops /// /// /// - /// /// - public static Tensor identity_n(Tensor input, TF_DataType[] T, string? name = null) + public static Tensor[] identity_n(Tensors input, string? name = null) { var _ctx = tf.Context; if (_ctx.executing_eagerly()) { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityN", name) { args = new object[] { input }, attrs = new Dictionary() { ["T"] = T } }); - return _fast_path_result[0]; + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityN", name) { args = new object[] { input }, attrs = new Dictionary() { } }); + return _fast_path_result; + } + catch (NotOkStatusException ex) + { + throw ex; } catch (Exception) { } try { - return identity_n_eager_fallback(input, T: T, name: name, ctx: _ctx); + return identity_n_eager_fallback(input, name: name, ctx: _ctx); } catch (Exception) { @@ -3154,7 +3326,6 @@ public static class gen_array_ops } Dictionary keywords = new(); keywords["input"] = input; - keywords["T"] = T; var _op = tf.OpDefLib._apply_op_helper("IdentityN", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) @@ -3162,19 +3333,19 @@ public static class gen_array_ops object[] _attrs = new object[] { "T", _op.get_attr("T") }; _execute.record_gradient("IdentityN", _op.inputs, _attrs, _result); } - return _result[0]; + return _result; } - public static Tensor identity_n_eager_fallback(Tensor input, TF_DataType[] T, string name, Context ctx) + public static Tensor[] identity_n_eager_fallback(Tensor input, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { input }; - object[] _attrs = new object[] { "T", T }; + object[] _attrs = new object[] { }; var _result = _execute.execute("IdentityN", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); if (_execute.must_record_gradient()) { _execute.record_gradient("IdentityN", _inputs_flat, _attrs, _result); } - return _result[0]; + return _result; } /// /// Returns immutable tensor from memory region. @@ -3211,6 +3382,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ImmutableConst", name) { args = new object[] { }, attrs = new Dictionary() { ["dtype"] = dtype, ["shape"] = shape, ["memory_region_name"] = memory_region_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3264,6 +3439,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InplaceAdd", name) { args = new object[] { x, i, v }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3317,6 +3496,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InplaceSub", name) { args = new object[] { x, i, v }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3370,6 +3553,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InplaceUpdate", name) { args = new object[] { x, i, v }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3440,6 +3627,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InvertPermutation", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3516,6 +3707,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ListDiff", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["out_idx"] = out_idx } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3590,6 +3785,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LowerBound", name) { args = new object[] { sorted_inputs, values }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3684,6 +3883,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixBandPart", name) { args = new object[] { input, num_lower, num_upper }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3765,6 +3968,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiag", name) { args = new object[] { diagonal }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3846,6 +4053,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiagPart", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3969,6 +4180,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiagPartV2", name) { args = new object[] { input, k, padding_value }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4136,6 +4351,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiagPartV3", name) { args = new object[] { input, k, padding_value }, attrs = new Dictionary() { ["align"] = align } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4287,6 +4506,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiagV2", name) { args = new object[] { diagonal, k, num_rows, num_cols, padding_value }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4475,6 +4698,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixDiagV3", name) { args = new object[] { diagonal, k, num_rows, num_cols, padding_value }, attrs = new Dictionary() { ["align"] = align } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4550,6 +4777,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixSetDiag", name) { args = new object[] { input, diagonal }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4677,6 +4908,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixSetDiagV2", name) { args = new object[] { input, diagonal, k }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4849,6 +5084,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatrixSetDiagV3", name) { args = new object[] { input, diagonal, k }, attrs = new Dictionary() { ["align"] = align } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4944,6 +5183,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MirrorPad", name) { args = new object[] { input, paddings }, attrs = new Dictionary() { ["mode"] = mode } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5023,6 +5266,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MirrorPadGrad", name) { args = new object[] { input, paddings }, attrs = new Dictionary() { ["mode"] = mode } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5173,6 +5420,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "OneHot", name) { args = new object[] { indices, depth, on_value, off_value }, attrs = new Dictionary() { ["axis"] = axis } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5226,6 +5477,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "OnesLike", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5304,6 +5559,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Pack", name) { args = new object[] { values }, attrs = new Dictionary() { ["axis"] = axis } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5384,6 +5643,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Pad", name) { args = new object[] { input, paddings }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5464,6 +5727,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PadV2", name) { args = new object[] { input, paddings, constant_values }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5541,6 +5808,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ParallelConcat", name) { args = new object[] { values }, attrs = new Dictionary() { ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5610,6 +5881,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Placeholder", name) { args = new object[] { }, attrs = new Dictionary() { ["dtype"] = dtype, ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5677,6 +5952,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PlaceholderV2", name) { args = new object[] { }, attrs = new Dictionary() { ["dtype"] = dtype, ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5732,6 +6011,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PlaceholderWithDefault", name) { args = new object[] { input }, attrs = new Dictionary() { ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5799,6 +6082,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PreventGradient", name) { args = new object[] { input }, attrs = new Dictionary() { ["message"] = message } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5858,6 +6145,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeAndDequantize", name) { args = new object[] { input }, attrs = new Dictionary() { ["signed_input"] = signed_input, ["num_bits"] = num_bits, ["range_given"] = range_given, ["input_min"] = input_min, ["input_max"] = input_max } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6011,6 +6302,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeAndDequantizeV2", name) { args = new object[] { input, input_min, input_max }, attrs = new Dictionary() { ["signed_input"] = signed_input, ["num_bits"] = num_bits, ["range_given"] = range_given, ["round_mode"] = round_mode, ["narrow_range"] = narrow_range, ["axis"] = axis } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6085,6 +6380,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeAndDequantizeV3", name) { args = new object[] { input, input_min, input_max, num_bits }, attrs = new Dictionary() { ["signed_input"] = signed_input, ["range_given"] = range_given, ["narrow_range"] = narrow_range, ["axis"] = axis } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6190,6 +6489,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeAndDequantizeV4", name) { args = new object[] { input, input_min, input_max }, attrs = new Dictionary() { ["signed_input"] = signed_input, ["num_bits"] = num_bits, ["range_given"] = range_given, ["round_mode"] = round_mode, ["narrow_range"] = narrow_range, ["axis"] = axis } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6387,6 +6690,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeV2", name) { args = new object[] { input, min_range, max_range }, attrs = new Dictionary() { ["T"] = T, ["mode"] = mode, ["round_mode"] = round_mode, ["narrow_range"] = narrow_range, ["axis"] = axis, ["ensure_minimum_range"] = ensure_minimum_range } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6455,6 +6762,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConcat", name) { args = new object[] { concat_dim, values, input_mins, input_maxes }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6541,6 +6852,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedInstanceNorm", name) { args = new object[] { x, x_min, x_max }, attrs = new Dictionary() { ["output_range_given"] = output_range_given, ["given_y_min"] = given_y_min, ["given_y_max"] = given_y_max, ["variance_epsilon"] = variance_epsilon, ["min_separation"] = min_separation } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6605,6 +6920,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedReshape", name) { args = new object[] { tensor, shape, input_min, input_max }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6674,6 +6993,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Rank", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6815,6 +7138,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Reshape", name) { args = new object[] { tensor, shape }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6884,6 +7211,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceStridedSliceAssign", name) { args = new object[] { ref_, begin, end, strides, value }, attrs = new Dictionary() { ["begin_mask"] = begin_mask, ["end_mask"] = end_mask, ["ellipsis_mask"] = ellipsis_mask, ["new_axis_mask"] = new_axis_mask, ["shrink_axis_mask"] = shrink_axis_mask } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6991,6 +7322,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Reverse", name) { args = new object[] { tensor, dims }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7110,6 +7445,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReverseSequence", name) { args = new object[] { input, seq_lengths }, attrs = new Dictionary() { ["seq_dim"] = seq_dim, ["batch_dim"] = batch_dim } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7210,6 +7549,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReverseV2", name) { args = new object[] { tensor, axis }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7352,6 +7695,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ScatterNd", name) { args = new object[] { indices, updates, shape }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7442,6 +7789,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ScatterNdNonAliasingAdd", name) { args = new object[] { input, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7506,6 +7857,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Shape", name) { args = new object[] { input }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7562,6 +7917,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ShapeN", name) { args = new object[] { input }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7628,6 +7987,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Size", name) { args = new object[] { input }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7690,6 +8053,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Slice", name) { args = new object[] { input, begin, size }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7741,6 +8108,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Snapshot", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7879,6 +8250,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SpaceToBatch", name) { args = new object[] { input, paddings }, attrs = new Dictionary() { ["block_size"] = block_size } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8048,6 +8423,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SpaceToBatchND", name) { args = new object[] { input, block_shape, paddings }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8192,6 +8571,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SpaceToDepth", name) { args = new object[] { input }, attrs = new Dictionary() { ["block_size"] = block_size, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8254,6 +8637,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Split", name) { args = new object[] { split_dim, value }, attrs = new Dictionary() { ["num_split"] = num_split } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8308,6 +8695,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SplitV", name) { args = new object[] { value, size_splits, split_dim }, attrs = new Dictionary() { ["num_split"] = num_split } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8393,6 +8784,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Squeeze", name) { args = new object[] { input }, attrs = new Dictionary() { ["squeeze_dims"] = squeeze_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8504,6 +8899,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StopGradient", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8689,6 +9088,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StridedSlice", name) { args = new object[] { input, begin, end, strides }, attrs = new Dictionary() { ["begin_mask"] = begin_mask, ["end_mask"] = end_mask, ["ellipsis_mask"] = ellipsis_mask, ["new_axis_mask"] = new_axis_mask, ["shrink_axis_mask"] = shrink_axis_mask } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8823,6 +9226,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StridedSliceGrad", name) { args = new object[] { shape, begin, end, strides, dy }, attrs = new Dictionary() { ["begin_mask"] = begin_mask, ["end_mask"] = end_mask, ["ellipsis_mask"] = ellipsis_mask, ["new_axis_mask"] = new_axis_mask, ["shrink_axis_mask"] = shrink_axis_mask } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8946,6 +9353,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorScatterAdd", name) { args = new object[] { tensor, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9013,6 +9424,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorScatterMax", name) { args = new object[] { tensor, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9066,6 +9481,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorScatterMin", name) { args = new object[] { tensor, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9185,6 +9604,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorScatterSub", name) { args = new object[] { tensor, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9278,6 +9701,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorScatterUpdate", name) { args = new object[] { tensor, indices, updates }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9348,6 +9775,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorStridedSliceUpdate", name) { args = new object[] { input, begin, end, strides, value }, attrs = new Dictionary() { ["begin_mask"] = begin_mask, ["end_mask"] = end_mask, ["ellipsis_mask"] = ellipsis_mask, ["new_axis_mask"] = new_axis_mask, ["shrink_axis_mask"] = shrink_axis_mask } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9437,6 +9868,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Tile", name) { args = new object[] { input, multiples }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9495,6 +9930,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TileGrad", name) { args = new object[] { input, multiples }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9552,6 +9991,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Transpose", name) { args = new object[] { x, perm }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9629,6 +10072,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Unique", name) { args = new object[] { x }, attrs = new Dictionary() { ["out_idx"] = out_idx } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9728,6 +10175,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UniqueV2", name) { args = new object[] { x, axis }, attrs = new Dictionary() { ["out_idx"] = out_idx } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9801,6 +10252,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UniqueWithCounts", name) { args = new object[] { x }, attrs = new Dictionary() { ["out_idx"] = out_idx } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9904,6 +10359,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UniqueWithCountsV2", name) { args = new object[] { x, axis }, attrs = new Dictionary() { ["out_idx"] = out_idx } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9978,6 +10437,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Unpack", name) { args = new object[] { value }, attrs = new Dictionary() { ["num"] = num, ["axis"] = axis } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -10054,6 +10517,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UnravelIndex", name) { args = new object[] { indices, dims }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -10127,6 +10594,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UpperBound", name) { args = new object[] { sorted_inputs, values }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -10241,6 +10712,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Where", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -10290,6 +10765,10 @@ public static class gen_array_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ZerosLike", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } diff --git a/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs b/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs index e1cf1c13..6ec426f5 100644 --- a/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_functional_ops.cs @@ -2,6 +2,7 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; @@ -54,6 +55,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Case", name) { args = new object[] { branch_index, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["branches"] = branches, ["output_shapes"] = output_shapes } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -115,6 +120,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DeviceIndex", name) { args = new object[] { }, attrs = new Dictionary() { ["device_names"] = device_names } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -172,6 +181,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FakeParam", name) { args = new object[] { }, attrs = new Dictionary() { ["dtype"] = dtype, ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -240,6 +253,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "For", name) { args = new object[] { start, limit, delta, input }, attrs = new Dictionary() { ["body"] = body } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -310,6 +327,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "If", name) { args = new object[] { cond, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["then_branch"] = then_branch, ["else_branch"] = else_branch, ["output_shapes"] = output_shapes } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -385,6 +406,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "PartitionedCall", name) { args = new object[] { args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f, ["config"] = config, ["config_proto"] = config_proto, ["executor_type"] = executor_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -462,6 +487,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RemoteCall", name) { args = new object[] { target, args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -529,6 +558,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatefulPartitionedCall", name) { args = new object[] { args }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f, ["config"] = config, ["config_proto"] = config_proto, ["executor_type"] = executor_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -628,6 +661,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessCase", name) { args = new object[] { branch_index, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["branches"] = branches, ["output_shapes"] = output_shapes } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -698,6 +735,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessIf", name) { args = new object[] { cond, input }, attrs = new Dictionary() { ["Tout"] = Tout, ["then_branch"] = then_branch, ["else_branch"] = else_branch, ["output_shapes"] = output_shapes } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -775,6 +816,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "StatelessWhile", name) { args = new object[] { input }, attrs = new Dictionary() { ["cond"] = cond, ["body"] = body, ["output_shapes"] = output_shapes, ["parallel_iterations"] = parallel_iterations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -855,6 +900,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SymbolicGradient", name) { args = new object[] { input }, attrs = new Dictionary() { ["Tout"] = Tout, ["f"] = f } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -922,6 +971,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ToBool", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -991,6 +1044,10 @@ public static class gen_functional_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "While", name) { args = new object[] { input }, attrs = new Dictionary() { ["cond"] = cond, ["body"] = body, ["output_shapes"] = output_shapes, ["parallel_iterations"] = parallel_iterations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } diff --git a/src/TensorFlowNET.Core/Operations/gen_io_ops.cs b/src/TensorFlowNET.Core/Operations/gen_io_ops.cs index 490cb188..0b92ff36 100644 --- a/src/TensorFlowNET.Core/Operations/gen_io_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_io_ops.cs @@ -2,12 +2,50 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; -internal static class gen_io_ops +public static class gen_io_ops { + /// + /// A Reader that outputs fixed-length records from a file. + /// + /// + /// + /// Number of bytes in the header, defaults to 0. + /// + /// + /// + /// + /// Number of bytes in the record. + /// + /// + /// + /// + /// Number of bytes in the footer, defaults to 0. + /// + /// + /// + /// + /// Number of bytes to hop before each read. Default of 0 means using + /// record_bytes. + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor fixed_length_record_reader(int header_bytes = 0, int record_bytes = 0, int footer_bytes = 0, int hop_bytes = 0, string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -15,9 +53,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FixedLengthRecordReader", name, "header_bytes", header_bytes, "record_bytes", record_bytes, "footer_bytes", footer_bytes, "hop_bytes", hop_bytes, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FixedLengthRecordReader", name) { args = new object[] { }, attrs = new Dictionary() { ["header_bytes"] = header_bytes, ["record_bytes"] = record_bytes, ["footer_bytes"] = footer_bytes, ["hop_bytes"] = hop_bytes, ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -29,8 +71,22 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["header_bytes"] = header_bytes; keywords["record_bytes"] = record_bytes; keywords["footer_bytes"] = footer_bytes; keywords["hop_bytes"] = hop_bytes; keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("FixedLengthRecordReader", name, keywords); + keywords["header_bytes"] = header_bytes; + keywords["record_bytes"] = record_bytes; + keywords["footer_bytes"] = footer_bytes; + keywords["hop_bytes"] = hop_bytes; + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("FixedLengthRecordReader", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -51,6 +107,49 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs fixed-length records from a file. + /// + /// + /// + /// Number of bytes in the header, defaults to 0. + /// + /// + /// + /// + /// Number of bytes in the record. + /// + /// + /// + /// + /// Number of bytes in the footer, defaults to 0. + /// + /// + /// + /// + /// Number of bytes to hop before each read. Default of 0 means using + /// record_bytes. + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// + /// + /// The type of encoding for the file. Currently ZLIB and GZIP + /// are supported. Defaults to none. + /// + /// + /// public static Tensor fixed_length_record_reader_v2(int header_bytes = 0, int record_bytes = 0, int footer_bytes = 0, int hop_bytes = 0, string container = "", string shared_name = "", string encoding = "", string? name = null) { var _ctx = tf.Context; @@ -58,9 +157,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FixedLengthRecordReaderV2", name, "header_bytes", header_bytes, "record_bytes", record_bytes, "footer_bytes", footer_bytes, "hop_bytes", hop_bytes, "container", container, "shared_name", shared_name, "encoding", encoding)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FixedLengthRecordReaderV2", name) { args = new object[] { }, attrs = new Dictionary() { ["header_bytes"] = header_bytes, ["record_bytes"] = record_bytes, ["footer_bytes"] = footer_bytes, ["hop_bytes"] = hop_bytes, ["container"] = container, ["shared_name"] = shared_name, ["encoding"] = encoding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -72,8 +175,27 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } + if (encoding is null) + { + encoding = ""; + } Dictionary keywords = new(); - keywords["header_bytes"] = header_bytes; keywords["record_bytes"] = record_bytes; keywords["footer_bytes"] = footer_bytes; keywords["hop_bytes"] = hop_bytes; keywords["container"] = container; keywords["shared_name"] = shared_name; keywords["encoding"] = encoding; var _op = tf.OpDefLib._apply_op_helper("FixedLengthRecordReaderV2", name, keywords); + keywords["header_bytes"] = header_bytes; + keywords["record_bytes"] = record_bytes; + keywords["footer_bytes"] = footer_bytes; + keywords["hop_bytes"] = hop_bytes; + keywords["container"] = container; + keywords["shared_name"] = shared_name; + keywords["encoding"] = encoding; + var _op = tf.OpDefLib._apply_op_helper("FixedLengthRecordReaderV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -94,6 +216,28 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the queued work as both the key and value. + /// + /// + /// + /// To use, enqueue strings in a Queue. ReaderRead will take the front + /// work string and output (work, work). + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor identity_reader(string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -101,9 +245,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityReader", name, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityReader", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -115,8 +263,18 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("IdentityReader", name, keywords); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("IdentityReader", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -137,6 +295,28 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the queued work as both the key and value. + /// + /// + /// + /// To use, enqueue strings in a Queue. ReaderRead will take the front + /// work string and output (work, work). + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor identity_reader_v2(string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -144,9 +324,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityReaderV2", name, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IdentityReaderV2", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -158,8 +342,18 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("IdentityReaderV2", name, keywords); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("IdentityReaderV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -180,6 +374,18 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Returns the set of files matching one or more glob patterns. + /// + /// + /// + /// Note that this routine only supports wildcard characters in the + /// basename portion of the pattern, not in the directory portion. + /// Note also that the order of filenames returned is deterministic. + /// + /// + /// + /// public static Tensor matching_files(Tensor pattern, string? name = null) { var _ctx = tf.Context; @@ -187,9 +393,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatchingFiles", name, pattern)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatchingFiles", name) { args = new object[] { pattern }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -224,51 +434,11 @@ internal static class gen_io_ops } return _result[0]; } - public static Operation merge_v2_checkpoints(Tensor checkpoint_prefixes, Tensor destination_prefix, bool delete_old_dirs = true, bool allow_missing_files = false, string? name = null) - { - var _ctx = tf.Context; - if (_ctx.executing_eagerly()) - { - try - { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MergeV2Checkpoints", name, checkpoint_prefixes, destination_prefix, "delete_old_dirs", delete_old_dirs, "allow_missing_files", allow_missing_files)); - return null; - } - catch (Exception) - { - } - try - { - return merge_v2_checkpoints_eager_fallback(checkpoint_prefixes, destination_prefix, delete_old_dirs: delete_old_dirs, allow_missing_files: allow_missing_files, name: name, ctx: _ctx); - } - catch (Exception) - { - } - } - Dictionary keywords = new(); - keywords["checkpoint_prefixes"] = checkpoint_prefixes; - keywords["destination_prefix"] = destination_prefix; - keywords["delete_old_dirs"] = delete_old_dirs; keywords["allow_missing_files"] = allow_missing_files; var _op = tf.OpDefLib._apply_op_helper("MergeV2Checkpoints", name, keywords); - var _result = _op.outputs; - if (_execute.must_record_gradient()) - { - object[] _attrs = new object[] { "delete_old_dirs", _op._get_attr_bool("delete_old_dirs"), "allow_missing_files", _op._get_attr_bool("allow_missing_files") }; - _execute.record_gradient("MergeV2Checkpoints", _op.inputs, _attrs, _result); - } - return _op; - } - - public static Tensor merge_v2_checkpoints_eager_fallback(Tensor checkpoint_prefixes, Tensor destination_prefix, bool delete_old_dirs, bool allow_missing_files, string name, Context ctx) - { - Tensor[] _inputs_flat = new Tensor[] { checkpoint_prefixes, destination_prefix }; - object[] _attrs = new object[] { "delete_old_dirs", delete_old_dirs, "allow_missing_files", allow_missing_files }; - var _result = _execute.execute("MergeV2Checkpoints", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); - if (_execute.must_record_gradient()) - { - _execute.record_gradient("MergeV2Checkpoints", _inputs_flat, _attrs, _result); - } - return null; - } + /// + /// Reads and outputs the entire contents of the input filename. + /// + /// + /// public static Tensor read_file(Tensor filename, string? name = null) { var _ctx = tf.Context; @@ -276,9 +446,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReadFile", name, filename)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReadFile", name) { args = new object[] { filename }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -313,6 +487,17 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Returns the number of records this Reader has produced. + /// + /// + /// + /// This is the same as the number of ReaderRead executions that have + /// succeeded. + /// + /// + /// + /// public static Tensor reader_num_records_produced(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -336,6 +521,17 @@ internal static class gen_io_ops { throw new RuntimeError($"reader_num_records_produced op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Returns the number of records this Reader has produced. + /// + /// + /// + /// This is the same as the number of ReaderRead executions that have + /// succeeded. + /// + /// + /// + /// public static Tensor reader_num_records_produced_v2(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -343,9 +539,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderNumRecordsProducedV2", name, reader_handle)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderNumRecordsProducedV2", name) { args = new object[] { reader_handle }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -380,6 +580,11 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Returns the number of work units this Reader has finished processing. + /// + /// + /// public static Tensor reader_num_work_units_completed(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -403,6 +608,11 @@ internal static class gen_io_ops { throw new RuntimeError($"reader_num_work_units_completed op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Returns the number of work units this Reader has finished processing. + /// + /// + /// public static Tensor reader_num_work_units_completed_v2(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -410,9 +620,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderNumWorkUnitsCompletedV2", name, reader_handle)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderNumWorkUnitsCompletedV2", name) { args = new object[] { reader_handle }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -447,6 +661,19 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Returns the next record (key, value pair) produced by a Reader. + /// + /// + /// + /// Will dequeue from the input queue if necessary (e.g. when the + /// Reader needs to start reading from a new file since it has finished + /// with the previous file). + /// + /// + /// + /// + /// public static Tensor[] reader_read(Tensor reader_handle, Tensor queue_handle, string? name = null) { var _ctx = tf.Context; @@ -471,6 +698,21 @@ internal static class gen_io_ops { throw new RuntimeError($"reader_read op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Returns up to `num_records` (key, value) pairs produced by a Reader. + /// + /// + /// + /// Will dequeue from the input queue if necessary (e.g. when the + /// Reader needs to start reading from a new file since it has finished + /// with the previous file). + /// It may return less than `num_records` even before the last batch. + /// + /// + /// + /// + /// + /// public static Tensor[] reader_read_up_to(Tensor reader_handle, Tensor queue_handle, Tensor num_records, string? name = null) { var _ctx = tf.Context; @@ -496,6 +738,21 @@ internal static class gen_io_ops { throw new RuntimeError($"reader_read_up_to op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Returns up to `num_records` (key, value) pairs produced by a Reader. + /// + /// + /// + /// Will dequeue from the input queue if necessary (e.g. when the + /// Reader needs to start reading from a new file since it has finished + /// with the previous file). + /// It may return less than `num_records` even before the last batch. + /// + /// + /// + /// + /// + /// public static Tensor[] reader_read_up_to_v2(Tensor reader_handle, Tensor queue_handle, Tensor num_records, string? name = null) { var _ctx = tf.Context; @@ -503,9 +760,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderReadUpToV2", name, reader_handle, queue_handle, num_records)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderReadUpToV2", name) { args = new object[] { reader_handle, queue_handle, num_records }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -542,6 +803,19 @@ internal static class gen_io_ops } return _result; } + /// + /// Returns the next record (key, value pair) produced by a Reader. + /// + /// + /// + /// Will dequeue from the input queue if necessary (e.g. when the + /// Reader needs to start reading from a new file since it has finished + /// with the previous file). + /// + /// + /// + /// + /// public static Tensor[] reader_read_v2(Tensor reader_handle, Tensor queue_handle, string? name = null) { var _ctx = tf.Context; @@ -549,9 +823,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderReadV2", name, reader_handle, queue_handle)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderReadV2", name) { args = new object[] { reader_handle, queue_handle }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -587,6 +865,11 @@ internal static class gen_io_ops } return _result; } + /// + /// Restore a Reader to its initial clean state. + /// + /// + /// public static Operation reader_reset(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -606,10 +889,15 @@ internal static class gen_io_ops return _op; } - public static Tensor reader_reset_eager_fallback(Tensor reader_handle, string name, Context ctx) + public static Operation reader_reset_eager_fallback(Tensor reader_handle, string name, Context ctx) { throw new RuntimeError($"reader_reset op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Restore a Reader to its initial clean state. + /// + /// + /// public static Operation reader_reset_v2(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -617,9 +905,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderResetV2", name, reader_handle)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderResetV2", name) { args = new object[] { reader_handle }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -643,7 +935,7 @@ internal static class gen_io_ops return _op; } - public static Tensor reader_reset_v2_eager_fallback(Tensor reader_handle, string name, Context ctx) + public static Operation reader_reset_v2_eager_fallback(Tensor reader_handle, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { reader_handle }; object[] _attrs = new object[] { }; @@ -654,6 +946,18 @@ internal static class gen_io_ops } return null; } + /// + /// Restore a reader to a previously saved state. + /// + /// + /// + /// Not all Readers support being restored, so this can produce an + /// Unimplemented error. + /// + /// + /// + /// + /// public static Operation reader_restore_state(Tensor reader_handle, Tensor state, string? name = null) { var _ctx = tf.Context; @@ -674,10 +978,22 @@ internal static class gen_io_ops return _op; } - public static Tensor reader_restore_state_eager_fallback(Tensor reader_handle, Tensor state, string name, Context ctx) + public static Operation reader_restore_state_eager_fallback(Tensor reader_handle, Tensor state, string name, Context ctx) { throw new RuntimeError($"reader_restore_state op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Restore a reader to a previously saved state. + /// + /// + /// + /// Not all Readers support being restored, so this can produce an + /// Unimplemented error. + /// + /// + /// + /// + /// public static Operation reader_restore_state_v2(Tensor reader_handle, Tensor state, string? name = null) { var _ctx = tf.Context; @@ -685,9 +1001,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderRestoreStateV2", name, reader_handle, state)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderRestoreStateV2", name) { args = new object[] { reader_handle, state }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -712,7 +1032,7 @@ internal static class gen_io_ops return _op; } - public static Tensor reader_restore_state_v2_eager_fallback(Tensor reader_handle, Tensor state, string name, Context ctx) + public static Operation reader_restore_state_v2_eager_fallback(Tensor reader_handle, Tensor state, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { reader_handle, state }; object[] _attrs = new object[] { }; @@ -723,6 +1043,17 @@ internal static class gen_io_ops } return null; } + /// + /// Produce a string tensor that encodes the state of a Reader. + /// + /// + /// + /// Not all Readers support being serialized, so this can produce an + /// Unimplemented error. + /// + /// + /// + /// public static Tensor reader_serialize_state(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -746,6 +1077,17 @@ internal static class gen_io_ops { throw new RuntimeError($"reader_serialize_state op does not support eager execution. Arg 'reader_handle' is a ref."); } + /// + /// Produce a string tensor that encodes the state of a Reader. + /// + /// + /// + /// Not all Readers support being serialized, so this can produce an + /// Unimplemented error. + /// + /// + /// + /// public static Tensor reader_serialize_state_v2(Tensor reader_handle, string? name = null) { var _ctx = tf.Context; @@ -753,9 +1095,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderSerializeStateV2", name, reader_handle)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReaderSerializeStateV2", name) { args = new object[] { reader_handle }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -790,6 +1136,43 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Restores a tensor from checkpoint files. + /// + /// + /// + /// Reads a tensor stored in one or several files. If there are several files (for + /// instance because a tensor was saved as slices), `file_pattern` may contain + /// wildcard symbols (`*` and `?`) in the filename portion only, not in the + /// directory portion. + /// + /// If a `file_pattern` matches several files, `preferred_shard` can be used to hint + /// in which file the requested tensor is likely to be found. This op will first + /// open the file at index `preferred_shard` in the list of matching files and try + /// to restore tensors from that file. Only if some tensors or tensor slices are + /// not found in that first file, then the Op opens all the files. Setting + /// `preferred_shard` to match the value passed as the `shard` input + /// of a matching `Save` Op may speed up Restore. This attribute only affects + /// performance, not correctness. The default value -1 means files are processed in + /// order. + /// + /// See also `RestoreSlice`. + /// + /// + /// + /// + /// + /// + /// The type of the tensor to be restored. + /// + /// + /// + /// + /// Index of file to open first if multiple files match + /// `file_pattern`. + /// + /// + /// public static Tensor restore(Tensor file_pattern, Tensor tensor_name, TF_DataType dt, int preferred_shard = -1, string? name = null) { var _ctx = tf.Context; @@ -797,9 +1180,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Restore", name, file_pattern, tensor_name, "dt", dt, "preferred_shard", preferred_shard)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Restore", name) { args = new object[] { file_pattern, tensor_name }, attrs = new Dictionary() { ["dt"] = dt, ["preferred_shard"] = preferred_shard } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -814,7 +1201,9 @@ internal static class gen_io_ops Dictionary keywords = new(); keywords["file_pattern"] = file_pattern; keywords["tensor_name"] = tensor_name; - keywords["dt"] = dt; keywords["preferred_shard"] = preferred_shard; var _op = tf.OpDefLib._apply_op_helper("Restore", name, keywords); + keywords["dt"] = dt; + keywords["preferred_shard"] = preferred_shard; + var _op = tf.OpDefLib._apply_op_helper("Restore", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -835,6 +1224,34 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Restores a tensor from checkpoint files. + /// + /// + /// + /// This is like `Restore` except that restored tensor can be listed as filling + /// only a slice of a larger tensor. `shape_and_slice` specifies the shape of the + /// larger tensor and the slice that the restored tensor covers. + /// + /// The `shape_and_slice` input has the same format as the + /// elements of the `shapes_and_slices` input of the `SaveSlices` op. + /// + /// + /// + /// + /// + /// + /// + /// The type of the tensor to be restored. + /// + /// + /// + /// + /// Index of file to open first if multiple files match + /// `file_pattern`. See the documentation for `Restore`. + /// + /// + /// public static Tensor restore_slice(Tensor file_pattern, Tensor tensor_name, Tensor shape_and_slice, TF_DataType dt, int preferred_shard = -1, string? name = null) { var _ctx = tf.Context; @@ -842,9 +1259,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RestoreSlice", name, file_pattern, tensor_name, shape_and_slice, "dt", dt, "preferred_shard", preferred_shard)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RestoreSlice", name) { args = new object[] { file_pattern, tensor_name, shape_and_slice }, attrs = new Dictionary() { ["dt"] = dt, ["preferred_shard"] = preferred_shard } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -860,7 +1281,9 @@ internal static class gen_io_ops keywords["file_pattern"] = file_pattern; keywords["tensor_name"] = tensor_name; keywords["shape_and_slice"] = shape_and_slice; - keywords["dt"] = dt; keywords["preferred_shard"] = preferred_shard; var _op = tf.OpDefLib._apply_op_helper("RestoreSlice", name, keywords); + keywords["dt"] = dt; + keywords["preferred_shard"] = preferred_shard; + var _op = tf.OpDefLib._apply_op_helper("RestoreSlice", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -881,15 +1304,49 @@ internal static class gen_io_ops } return _result[0]; } - public static Tensor restore_v2(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, TF_DataType[] dtypes, string? name = null) + /// + /// Restores tensors from a V2 checkpoint. + /// + /// + /// + /// For backward compatibility with the V1 format, this Op currently allows + /// restoring from a V1 checkpoint as well: + /// - This Op first attempts to find the V2 index file pointed to by "prefix", and + /// if found proceed to read it as a V2 checkpoint; + /// - Otherwise the V1 read path is invoked. + /// Relying on this behavior is not recommended, as the ability to fall back to read + /// V1 might be deprecated and eventually removed. + /// + /// By default, restores the named tensors in full. If the caller wishes to restore + /// specific slices of stored tensors, "shape_and_slices" should be non-empty + /// strings and correspondingly well-formed. + /// + /// Callers must ensure all the named tensors are indeed stored in the checkpoint. + /// + /// + /// + /// + /// + /// + /// + /// shape {N}. The list of expected dtype for the tensors. Must match + /// those stored in the checkpoint. + /// + /// + /// + public static Tensor[] restore_v2(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, TF_DataType[] dtypes, string? name = null) { var _ctx = tf.Context; if (_ctx.executing_eagerly()) { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RestoreV2", name, prefix, tensor_names, shape_and_slices, "dtypes", dtypes)); - return _fast_path_result[0]; + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RestoreV2", name) { args = new object[] { prefix, tensor_names, shape_and_slices }, attrs = new Dictionary() { ["dtypes"] = dtypes } }); + return _fast_path_result; + } + catch (NotOkStatusException ex) + { + throw ex; } catch (Exception) { @@ -906,43 +1363,63 @@ internal static class gen_io_ops keywords["prefix"] = prefix; keywords["tensor_names"] = tensor_names; keywords["shape_and_slices"] = shape_and_slices; - keywords["dtypes"] = dtypes; var _op = tf.OpDefLib._apply_op_helper("RestoreV2", name, keywords); + keywords["dtypes"] = dtypes; + var _op = tf.OpDefLib._apply_op_helper("RestoreV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { object[] _attrs = new object[] { "dtypes", _op.get_attr("dtypes") }; _execute.record_gradient("RestoreV2", _op.inputs, _attrs, _result); } - return _result[0]; + return _result; } - public static Tensor restore_v2_eager_fallback(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, TF_DataType[] dtypes, string name, Context ctx) + public static Tensor[] restore_v2_eager_fallback(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, TF_DataType[] dtypes, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { prefix, tensor_names, shape_and_slices }; - object[] _attrs = new object[] { "dtypes", dtypes }; + object[] _attrs = new object[] { }; var _result = _execute.execute("RestoreV2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); if (_execute.must_record_gradient()) { _execute.record_gradient("RestoreV2", _inputs_flat, _attrs, _result); } - return _result[0]; + return _result; } - public static Operation save(Tensor filename, Tensor tensor_names, Tensor data, TF_DataType[] T, string? name = null) + /// + /// Saves the input tensors to disk. + /// + /// + /// + /// The size of `tensor_names` must match the number of tensors in `data`. `data[i]` + /// is written to `filename` with name `tensor_names[i]`. + /// + /// See also `SaveSlices`. + /// + /// + /// + /// + /// + /// + public static Operation save(Tensor filename, Tensor tensor_names, Tensors data, string? name = null) { var _ctx = tf.Context; if (_ctx.executing_eagerly()) { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Save", name, filename, tensor_names, data, "T", T)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Save", name) { args = new object[] { filename, tensor_names, data }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } try { - return save_eager_fallback(filename, tensor_names, data, T: T, name: name, ctx: _ctx); + return save_eager_fallback(filename, tensor_names, data, name: name, ctx: _ctx); } catch (Exception) { @@ -952,7 +1429,7 @@ internal static class gen_io_ops keywords["filename"] = filename; keywords["tensor_names"] = tensor_names; keywords["data"] = data; - keywords["T"] = T; var _op = tf.OpDefLib._apply_op_helper("Save", name, keywords); + var _op = tf.OpDefLib._apply_op_helper("Save", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -962,10 +1439,10 @@ internal static class gen_io_ops return _op; } - public static Tensor save_eager_fallback(Tensor filename, Tensor tensor_names, Tensor data, TF_DataType[] T, string name, Context ctx) + public static Operation save_eager_fallback(Tensor filename, Tensor tensor_names, Tensor data, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { filename, tensor_names, data }; - object[] _attrs = new object[] { "T", T }; + object[] _attrs = new object[] { }; var _result = _execute.execute("Save", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); if (_execute.must_record_gradient()) { @@ -973,22 +1450,59 @@ internal static class gen_io_ops } return null; } - public static Operation save_slices(Tensor filename, Tensor tensor_names, Tensor shapes_and_slices, Tensor data, TF_DataType[] T, string? name = null) + /// + /// Saves input tensors slices to disk. + /// + /// + /// + /// This is like `Save` except that tensors can be listed in the saved file as being + /// a slice of a larger tensor. `shapes_and_slices` specifies the shape of the + /// larger tensor and the slice that this tensor covers. `shapes_and_slices` must + /// have as many elements as `tensor_names`. + /// + /// Elements of the `shapes_and_slices` input must either be: + /// + /// * The empty string, in which case the corresponding tensor is + /// saved normally. + /// * A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the + /// `dimI` are the dimensions of the larger tensor and `slice-spec` + /// specifies what part is covered by the tensor to save. + /// + /// `slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1` + /// where each `sliceI` is either: + /// + /// * The string `-` meaning that the slice covers all indices of this dimension + /// * `start,length` where `start` and `length` are integers. In that + /// case the slice covers `length` indices starting at `start`. + /// + /// See also `Save`. + /// + /// + /// + /// + /// + /// + /// + public static Operation save_slices(Tensor filename, Tensor tensor_names, Tensor shapes_and_slices, Tensors data, string? name = null) { var _ctx = tf.Context; if (_ctx.executing_eagerly()) { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SaveSlices", name, filename, tensor_names, shapes_and_slices, data, "T", T)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SaveSlices", name) { args = new object[] { filename, tensor_names, shapes_and_slices, data }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } try { - return save_slices_eager_fallback(filename, tensor_names, shapes_and_slices, data, T: T, name: name, ctx: _ctx); + return save_slices_eager_fallback(filename, tensor_names, shapes_and_slices, data, name: name, ctx: _ctx); } catch (Exception) { @@ -999,7 +1513,7 @@ internal static class gen_io_ops keywords["tensor_names"] = tensor_names; keywords["shapes_and_slices"] = shapes_and_slices; keywords["data"] = data; - keywords["T"] = T; var _op = tf.OpDefLib._apply_op_helper("SaveSlices", name, keywords); + var _op = tf.OpDefLib._apply_op_helper("SaveSlices", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1009,10 +1523,10 @@ internal static class gen_io_ops return _op; } - public static Tensor save_slices_eager_fallback(Tensor filename, Tensor tensor_names, Tensor shapes_and_slices, Tensor data, TF_DataType[] T, string name, Context ctx) + public static Operation save_slices_eager_fallback(Tensor filename, Tensor tensor_names, Tensor shapes_and_slices, Tensor data, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { filename, tensor_names, shapes_and_slices, data }; - object[] _attrs = new object[] { "T", T }; + object[] _attrs = new object[] { }; var _result = _execute.execute("SaveSlices", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); if (_execute.must_record_gradient()) { @@ -1020,22 +1534,41 @@ internal static class gen_io_ops } return null; } - public static Operation save_v2(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, Tensor tensors, TF_DataType[] dtypes, string? name = null) + /// + /// Saves tensors in V2 checkpoint format. + /// + /// + /// + /// By default, saves the named tensors in full. If the caller wishes to save + /// specific slices of full tensors, "shape_and_slices" should be non-empty strings + /// and correspondingly well-formed. + /// + /// + /// + /// + /// + /// + /// + public static Operation save_v2(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, Tensors tensors, string? name = null) { var _ctx = tf.Context; if (_ctx.executing_eagerly()) { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SaveV2", name, prefix, tensor_names, shape_and_slices, tensors, "dtypes", dtypes)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SaveV2", name) { args = new object[] { prefix, tensor_names, shape_and_slices, tensors }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } try { - return save_v2_eager_fallback(prefix, tensor_names, shape_and_slices, tensors, dtypes: dtypes, name: name, ctx: _ctx); + return save_v2_eager_fallback(prefix, tensor_names, shape_and_slices, tensors, name: name, ctx: _ctx); } catch (Exception) { @@ -1046,7 +1579,7 @@ internal static class gen_io_ops keywords["tensor_names"] = tensor_names; keywords["shape_and_slices"] = shape_and_slices; keywords["tensors"] = tensors; - keywords["dtypes"] = dtypes; var _op = tf.OpDefLib._apply_op_helper("SaveV2", name, keywords); + var _op = tf.OpDefLib._apply_op_helper("SaveV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1056,10 +1589,10 @@ internal static class gen_io_ops return _op; } - public static Tensor save_v2_eager_fallback(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, Tensor tensors, TF_DataType[] dtypes, string name, Context ctx) + public static Operation save_v2_eager_fallback(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, Tensor tensors, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { prefix, tensor_names, shape_and_slices, tensors }; - object[] _attrs = new object[] { "dtypes", dtypes }; + object[] _attrs = new object[] { }; var _result = _execute.execute("SaveV2", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); if (_execute.must_record_gradient()) { @@ -1067,6 +1600,18 @@ internal static class gen_io_ops } return null; } + /// + /// Generate a sharded filename. The filename is printf formatted as + /// + /// + /// + /// %s-%05d-of-%05d, basename, shard, num_shards. + /// + /// + /// + /// + /// + /// public static Tensor sharded_filename(Tensor basename, Tensor shard, Tensor num_shards, string? name = null) { var _ctx = tf.Context; @@ -1074,9 +1619,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ShardedFilename", name, basename, shard, num_shards)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ShardedFilename", name) { args = new object[] { basename, shard, num_shards }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1113,6 +1662,12 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Generate a glob pattern matching all sharded file names. + /// + /// + /// + /// public static Tensor sharded_filespec(Tensor basename, Tensor num_shards, string? name = null) { var _ctx = tf.Context; @@ -1120,9 +1675,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ShardedFilespec", name, basename, num_shards)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ShardedFilespec", name) { args = new object[] { basename, num_shards }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1158,6 +1717,27 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the lines of a file delimited by '\n'. + /// + /// + /// + /// Number of lines to skip from the beginning of every file. + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor text_line_reader(int skip_header_lines = 0, string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -1165,9 +1745,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TextLineReader", name, "skip_header_lines", skip_header_lines, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TextLineReader", name) { args = new object[] { }, attrs = new Dictionary() { ["skip_header_lines"] = skip_header_lines, ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1179,8 +1763,19 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["skip_header_lines"] = skip_header_lines; keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("TextLineReader", name, keywords); + keywords["skip_header_lines"] = skip_header_lines; + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("TextLineReader", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1201,6 +1796,27 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the lines of a file delimited by '\n'. + /// + /// + /// + /// Number of lines to skip from the beginning of every file. + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor text_line_reader_v2(int skip_header_lines = 0, string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -1208,9 +1824,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TextLineReaderV2", name, "skip_header_lines", skip_header_lines, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TextLineReaderV2", name) { args = new object[] { }, attrs = new Dictionary() { ["skip_header_lines"] = skip_header_lines, ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1222,8 +1842,19 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["skip_header_lines"] = skip_header_lines; keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("TextLineReaderV2", name, keywords); + keywords["skip_header_lines"] = skip_header_lines; + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("TextLineReaderV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1244,6 +1875,28 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the entire contents of a file as a value. + /// + /// + /// + /// To use, enqueue filenames in a Queue. The output of ReaderRead will + /// be a filename (key) and the contents of that file (value). + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor whole_file_reader(string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -1251,9 +1904,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WholeFileReader", name, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WholeFileReader", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1265,8 +1922,18 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("WholeFileReader", name, keywords); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("WholeFileReader", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1287,6 +1954,28 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// A Reader that outputs the entire contents of a file as a value. + /// + /// + /// + /// To use, enqueue filenames in a Queue. The output of ReaderRead will + /// be a filename (key) and the contents of that file (value). + /// + /// + /// + /// + /// If non-empty, this reader is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this reader is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// public static Tensor whole_file_reader_v2(string container = "", string shared_name = "", string? name = null) { var _ctx = tf.Context; @@ -1294,9 +1983,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WholeFileReaderV2", name, "container", container, "shared_name", shared_name)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WholeFileReaderV2", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1308,8 +2001,18 @@ internal static class gen_io_ops { } } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } Dictionary keywords = new(); - keywords["container"] = container; keywords["shared_name"] = shared_name; var _op = tf.OpDefLib._apply_op_helper("WholeFileReaderV2", name, keywords); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("WholeFileReaderV2", name, keywords); var _result = _op.outputs; if (_execute.must_record_gradient()) { @@ -1330,6 +2033,17 @@ internal static class gen_io_ops } return _result[0]; } + /// + /// Writes `contents` to the file at input `filename`. + /// + /// + /// + /// Creates the file and recursively creates directory if it does not exist. + /// + /// + /// + /// + /// public static Operation write_file(Tensor filename, Tensor contents, string? name = null) { var _ctx = tf.Context; @@ -1337,9 +2051,13 @@ internal static class gen_io_ops { try { - var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WriteFile", name, filename, contents)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "WriteFile", name) { args = new object[] { filename, contents }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1364,7 +2082,7 @@ internal static class gen_io_ops return _op; } - public static Tensor write_file_eager_fallback(Tensor filename, Tensor contents, string name, Context ctx) + public static Operation write_file_eager_fallback(Tensor filename, Tensor contents, string name, Context ctx) { Tensor[] _inputs_flat = new Tensor[] { filename, contents }; object[] _attrs = new object[] { }; diff --git a/src/TensorFlowNET.Core/Operations/gen_list_ops.cs b/src/TensorFlowNET.Core/Operations/gen_list_ops.cs index e7253986..59c783b2 100644 --- a/src/TensorFlowNET.Core/Operations/gen_list_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_list_ops.cs @@ -2,6 +2,7 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; @@ -35,6 +36,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EmptyTensorList", name) { args = new object[] { element_shape, max_num_elements }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -98,6 +103,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcat", name) { args = new object[] { input_handle }, attrs = new Dictionary() { ["element_dtype"] = element_dtype, ["element_shape"] = element_shape } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -151,6 +160,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcatLists", name) { args = new object[] { input_a, input_b }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -221,6 +234,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListConcatV2", name) { args = new object[] { input_handle, element_shape, leading_dims }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -280,6 +297,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListElementShape", name) { args = new object[] { input_handle }, attrs = new Dictionary() { ["shape_type"] = shape_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -339,6 +360,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListFromTensor", name) { args = new object[] { tensor, element_shape }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -402,6 +427,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListGather", name) { args = new object[] { input_handle, indices, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -457,6 +486,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListGetItem", name) { args = new object[] { input_handle, index, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -515,6 +548,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListLength", name) { args = new object[] { input_handle }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -576,6 +613,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPopBack", name) { args = new object[] { input_handle, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -637,6 +678,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPushBack", name) { args = new object[] { input_handle, tensor }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -688,6 +733,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListPushBackBatch", name) { args = new object[] { input_handles, tensor }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -748,6 +797,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListReserve", name) { args = new object[] { element_shape, num_elements }, attrs = new Dictionary() { ["element_dtype"] = element_dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -808,6 +861,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListResize", name) { args = new object[] { input_handle, size }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -872,6 +929,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatter", name) { args = new object[] { tensor, indices, element_shape }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -936,6 +997,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatterIntoExistingList", name) { args = new object[] { input_handle, tensor, indices }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1005,6 +1070,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListScatterV2", name) { args = new object[] { tensor, indices, element_shape, num_elements }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1059,6 +1128,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListSetItem", name) { args = new object[] { input_handle, index, item }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1123,6 +1196,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListSplit", name) { args = new object[] { tensor, element_shape, lengths }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1187,6 +1264,10 @@ public static class gen_list_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TensorListStack", name) { args = new object[] { input_handle, element_shape }, attrs = new Dictionary() { ["element_dtype"] = element_dtype, ["num_elements"] = num_elements } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index 6eb7a411..a8152a11 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -2,6 +2,7 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; @@ -30,6 +31,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Abs", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -96,6 +101,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AccumulateNV2", name) { args = new object[] { inputs }, attrs = new Dictionary() { ["shape"] = shape } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -157,6 +166,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Acos", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -217,6 +230,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Acosh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -278,6 +295,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Add", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -338,6 +359,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AddN", name) { args = new object[] { inputs }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -396,6 +421,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AddV2", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -460,6 +489,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "All", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -533,6 +566,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Angle", name) { args = new object[] { input }, attrs = new Dictionary() { ["Tout"] = Tout } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -597,6 +634,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Any", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -650,6 +691,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ApproximateEqual", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["tolerance"] = tolerance } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -718,6 +763,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ArgMax", name) { args = new object[] { input, dimension }, attrs = new Dictionary() { ["output_type"] = output_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -786,6 +835,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ArgMin", name) { args = new object[] { input, dimension }, attrs = new Dictionary() { ["output_type"] = output_type } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -857,6 +910,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Asin", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -918,6 +975,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Asinh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -987,6 +1048,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Atan", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1055,6 +1120,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Atan2", name) { args = new object[] { y, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1119,6 +1188,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Atanh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1201,6 +1274,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatMul", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["adj_x"] = adj_x, ["adj_y"] = adj_y } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1291,6 +1368,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatMulV2", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["adj_x"] = adj_x, ["adj_y"] = adj_y } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1386,6 +1467,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchMatMulV3", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["Tout"] = Tout, ["adj_x"] = adj_x, ["adj_y"] = adj_y } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1458,6 +1543,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Betainc", name) { args = new object[] { a, b, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1522,6 +1611,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Bincount", name) { args = new object[] { arr, size, weights }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1592,6 +1685,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Bucketize", name) { args = new object[] { input }, attrs = new Dictionary() { ["boundaries"] = boundaries } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1644,6 +1741,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cast", name) { args = new object[] { x }, attrs = new Dictionary() { ["DstT"] = DstT, ["Truncate"] = Truncate } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1695,6 +1796,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Ceil", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1754,6 +1859,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ClipByValue", name) { args = new object[] { t, clip_value_min, clip_value_max }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1825,6 +1934,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Complex", name) { args = new object[] { real, imag }, attrs = new Dictionary() { ["Tout"] = Tout } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1892,6 +2005,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ComplexAbs", name) { args = new object[] { x }, attrs = new Dictionary() { ["Tout"] = Tout } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1959,6 +2076,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conj", name) { args = new object[] { input }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2021,6 +2142,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cos", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2082,6 +2207,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cosh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2139,6 +2268,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cross", name) { args = new object[] { a, b }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2232,6 +2365,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cumprod", name) { args = new object[] { x, axis }, attrs = new Dictionary() { ["exclusive"] = exclusive, ["reverse"] = reverse } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2327,6 +2464,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Cumsum", name) { args = new object[] { x, axis }, attrs = new Dictionary() { ["exclusive"] = exclusive, ["reverse"] = reverse } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2412,6 +2553,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CumulativeLogsumexp", name) { args = new object[] { x, axis }, attrs = new Dictionary() { ["exclusive"] = exclusive, ["reverse"] = reverse } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2482,6 +2627,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DenseBincount", name) { args = new object[] { input, size, weights }, attrs = new Dictionary() { ["binary_output"] = binary_output } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2539,6 +2688,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Digamma", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2595,6 +2748,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Div", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2653,6 +2810,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DivNoNan", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2721,6 +2882,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Equal", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["incompatible_shape_error"] = incompatible_shape_error } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2772,6 +2937,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Erf", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2821,6 +2990,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Erfc", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2870,6 +3043,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Erfinv", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2933,6 +3110,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EuclideanNorm", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3014,6 +3195,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Exp", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3080,6 +3265,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Expm1", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3129,6 +3318,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Floor", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3185,6 +3378,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FloorDiv", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3246,6 +3443,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FloorMod", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3315,6 +3516,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Greater", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3384,6 +3589,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "GreaterEqual", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3456,6 +3665,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "HistogramFixedWidth", name) { args = new object[] { values, value_range, nbins }, attrs = new Dictionary() { ["dtype"] = dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3526,6 +3739,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Igamma", name) { args = new object[] { a, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3577,6 +3794,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IgammaGradA", name) { args = new object[] { a, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3644,6 +3865,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Igammac", name) { args = new object[] { a, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3710,6 +3935,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Imag", name) { args = new object[] { input }, attrs = new Dictionary() { ["Tout"] = Tout } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3765,6 +3994,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Inv", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3821,6 +4054,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InvGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3885,6 +4122,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IsFinite", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3948,6 +4189,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IsInf", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4011,6 +4256,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IsNan", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4079,6 +4328,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Less", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4148,6 +4401,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LessEqual", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4211,6 +4468,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Lgamma", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4275,6 +4536,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LinSpace", name) { args = new object[] { start, stop, num }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4338,6 +4603,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Log", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4399,6 +4668,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Log1p", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4455,6 +4728,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LogicalAnd", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4505,6 +4782,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LogicalNot", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4561,6 +4842,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LogicalOr", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4633,9 +4918,12 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MatMul", name) { args = new object[] { a, b }, attrs = new Dictionary() { ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b } }); return _fast_path_result[0]; } - catch (Exception ex) + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) { - Console.WriteLine(); } try { @@ -4700,6 +4988,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Max", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4758,6 +5050,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Maximum", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4822,6 +5118,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Mean", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4887,6 +5187,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Min", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4945,6 +5249,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Minimum", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5005,6 +5313,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Mod", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5062,6 +5374,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Mul", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5119,6 +5435,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MulNoNan", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5169,6 +5489,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Ndtri", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5223,6 +5547,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Neg", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5284,6 +5612,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "NextAfter", name) { args = new object[] { x1, x2 }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5342,6 +5674,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "NotEqual", name) { args = new object[] { x, y }, attrs = new Dictionary() { ["incompatible_shape_error"] = incompatible_shape_error } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5405,6 +5741,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Polygamma", name) { args = new object[] { a, x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5468,6 +5808,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Pow", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5532,6 +5876,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Prod", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5616,6 +5964,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizeDownAndShrinkRange", name) { args = new object[] { input, input_min, input_max }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5674,6 +6026,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedAdd", name) { args = new object[] { x, y, min_x, max_x, min_y, max_y }, attrs = new Dictionary() { ["Toutput"] = Toutput } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5759,6 +6115,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMul", name) { args = new object[] { a, b, min_a, max_a, min_b, max_b }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["Tactivation"] = Tactivation } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5823,6 +6183,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMul", name) { args = new object[] { x, y, min_x, max_x, min_y, max_y }, attrs = new Dictionary() { ["Toutput"] = Toutput } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5897,6 +6261,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RaggedBincount", name) { args = new object[] { splits, values, size, weights }, attrs = new Dictionary() { ["binary_output"] = binary_output } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5967,6 +6335,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Range", name) { args = new object[] { start, limit, delta }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6034,6 +6406,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Real", name) { args = new object[] { input }, attrs = new Dictionary() { ["Tout"] = Tout } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6093,6 +6469,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RealDiv", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6148,6 +6528,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Reciprocal", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6204,6 +6588,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReciprocalGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6264,6 +6652,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RequantizationRange", name) { args = new object[] { input, input_min, input_max }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6323,6 +6715,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RequantizationRangePerChannel", name) { args = new object[] { input, input_min, input_max }, attrs = new Dictionary() { ["clip_value_max"] = clip_value_max } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6395,6 +6791,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Requantize", name) { args = new object[] { input, input_min, input_max, requested_output_min, requested_output_max }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6458,6 +6858,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RequantizePerChannel", name) { args = new object[] { input, input_min, input_max, requested_output_min, requested_output_max }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6525,6 +6929,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Rint", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6580,6 +6988,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Round", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6634,6 +7046,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Rsqrt", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6690,6 +7106,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "RsqrtGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6772,6 +7192,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SegmentMax", name) { args = new object[] { data, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6856,6 +7280,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SegmentMean", name) { args = new object[] { data, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6938,6 +7366,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SegmentMin", name) { args = new object[] { data, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7020,6 +7452,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SegmentProd", name) { args = new object[] { data, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7102,6 +7538,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SegmentSum", name) { args = new object[] { data, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7196,6 +7636,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Select", name) { args = new object[] { condition, t, e }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7249,6 +7693,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SelectV2", name) { args = new object[] { condition, t, e }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7305,6 +7753,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sigmoid", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7361,6 +7813,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SigmoidGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7422,6 +7878,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sign", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7483,6 +7943,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sin", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7544,6 +8008,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sinh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7606,6 +8074,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SobolSample", name) { args = new object[] { dim, num_results, skip }, attrs = new Dictionary() { ["dtype"] = dtype } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7678,6 +8150,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseBincount", name) { args = new object[] { indices, values, dense_shape, size, weights }, attrs = new Dictionary() { ["binary_output"] = binary_output } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7750,6 +8226,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseMatMul", name) { args = new object[] { a, b }, attrs = new Dictionary() { ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["a_is_sparse"] = a_is_sparse, ["b_is_sparse"] = b_is_sparse } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7814,6 +8294,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentMean", name) { args = new object[] { data, indices, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7874,6 +8358,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentMeanGrad", name) { args = new object[] { grad, indices, segment_ids, output_dim0 }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7939,6 +8427,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentMeanWithNumSegments", name) { args = new object[] { data, indices, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8001,6 +8493,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentSqrtN", name) { args = new object[] { data, indices, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8087,6 +8583,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentSum", name) { args = new object[] { data, indices, segment_ids }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8147,6 +8647,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentSumGrad", name) { args = new object[] { grad, indices, segment_ids, output_dim0 }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8233,6 +8737,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSegmentSumWithNumSegments", name) { args = new object[] { data, indices, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8290,6 +8798,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sqrt", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8346,6 +8858,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SqrtGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8401,6 +8917,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Square", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8457,6 +8977,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SquaredDifference", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8514,6 +9038,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sub", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8578,6 +9106,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Sum", name) { args = new object[] { input, reduction_indices }, attrs = new Dictionary() { ["keep_dims"] = keep_dims } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8642,6 +9174,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Tan", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8705,6 +9241,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Tanh", name) { args = new object[] { x }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8761,6 +9301,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TanhGrad", name) { args = new object[] { y, dy }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8823,6 +9367,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TruncateDiv", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8883,6 +9431,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TruncateMod", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8974,6 +9526,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UnsortedSegmentMax", name) { args = new object[] { data, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9061,6 +9617,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UnsortedSegmentMin", name) { args = new object[] { data, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9147,6 +9707,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UnsortedSegmentProd", name) { args = new object[] { data, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9237,6 +9801,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "UnsortedSegmentSum", name) { args = new object[] { data, segment_ids, num_segments }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9289,6 +9857,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Xdivy", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9340,6 +9912,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Xlog1py", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9391,6 +9967,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Xlogy", name) { args = new object[] { x, y }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -9450,6 +10030,10 @@ public static class gen_math_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Zeta", name) { args = new object[] { x, q }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } diff --git a/src/TensorFlowNET.Core/Operations/gen_nn_ops.cs b/src/TensorFlowNET.Core/Operations/gen_nn_ops.cs index c0cec278..59c740c4 100644 --- a/src/TensorFlowNET.Core/Operations/gen_nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_nn_ops.cs @@ -2,6 +2,7 @@ using Tensorflow.Eager; using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; namespace Tensorflow; @@ -57,6 +58,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ApproxTopK", name) { args = new object[] { input }, attrs = new Dictionary() { ["k"] = k, ["reduction_dimension"] = reduction_dimension, ["recall_target"] = recall_target, ["is_max_k"] = is_max_k, ["reduction_input_size_override"] = reduction_input_size_override, ["aggregate_to_topk"] = aggregate_to_topk } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -142,6 +147,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AvgPool", name) { args = new object[] { value }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -231,6 +240,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AvgPool3D", name) { args = new object[] { input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -315,6 +328,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AvgPool3DGrad", name) { args = new object[] { orig_input_shape, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -398,6 +415,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AvgPoolGrad", name) { args = new object[] { orig_input_shape, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -476,6 +497,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchNormWithGlobalNormalization", name) { args = new object[] { t, m, v, beta, gamma }, attrs = new Dictionary() { ["variance_epsilon"] = variance_epsilon, ["scale_after_normalization"] = scale_after_normalization } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -551,6 +576,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BatchNormWithGlobalNormalizationGrad", name) { args = new object[] { t, m, v, gamma, backprop }, attrs = new Dictionary() { ["variance_epsilon"] = variance_epsilon, ["scale_after_normalization"] = scale_after_normalization } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -624,6 +653,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BiasAdd", name) { args = new object[] { value, bias }, attrs = new Dictionary() { ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -697,6 +730,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BiasAddGrad", name) { args = new object[] { out_backprop }, attrs = new Dictionary() { ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -760,6 +797,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "BiasAddV1", name) { args = new object[] { value, bias }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -883,6 +924,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv2D", name) { args = new object[] { input, filter }, attrs = new Dictionary() { ["strides"] = strides, ["use_cudnn_on_gpu"] = use_cudnn_on_gpu, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -992,6 +1037,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv2DBackpropFilter", name) { args = new object[] { input, filter_sizes, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["use_cudnn_on_gpu"] = use_cudnn_on_gpu, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1102,6 +1151,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv2DBackpropInput", name) { args = new object[] { input_sizes, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["use_cudnn_on_gpu"] = use_cudnn_on_gpu, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1206,6 +1259,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv3D", name) { args = new object[] { input, filter }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1282,6 +1339,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv3DBackpropFilter", name) { args = new object[] { input, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1371,6 +1432,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv3DBackpropFilterV2", name) { args = new object[] { input, filter_sizes, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1448,6 +1513,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv3DBackpropInput", name) { args = new object[] { input, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1537,6 +1606,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Conv3DBackpropInputV2", name) { args = new object[] { input_sizes, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1611,6 +1684,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DataFormatDimMap", name) { args = new object[] { x }, attrs = new Dictionary() { ["src_format"] = src_format, ["dst_format"] = dst_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1715,6 +1792,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DataFormatVecPermute", name) { args = new object[] { x }, attrs = new Dictionary() { ["src_format"] = src_format, ["dst_format"] = dst_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1835,6 +1916,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DepthwiseConv2dNative", name) { args = new object[] { input, filter }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -1934,6 +2019,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DepthwiseConv2dNativeBackpropFilter", name) { args = new object[] { input, filter_sizes, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2034,6 +2123,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DepthwiseConv2dNativeBackpropInput", name) { args = new object[] { input_sizes, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format, ["dilations"] = dilations } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2139,6 +2232,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Dilation2D", name) { args = new object[] { input, filter }, attrs = new Dictionary() { ["strides"] = strides, ["rates"] = rates, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2211,6 +2308,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Dilation2DBackpropFilter", name) { args = new object[] { input, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["rates"] = rates, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2284,6 +2385,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Dilation2DBackpropInput", name) { args = new object[] { input, filter, out_backprop }, attrs = new Dictionary() { ["strides"] = strides, ["rates"] = rates, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2358,6 +2463,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Elu", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2408,6 +2517,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "EluGrad", name) { args = new object[] { gradients, outputs }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2516,6 +2629,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FractionalAvgPool", name) { args = new object[] { value }, attrs = new Dictionary() { ["pooling_ratio"] = pooling_ratio, ["pseudo_random"] = pseudo_random, ["overlapping"] = overlapping, ["deterministic"] = deterministic, ["seed"] = seed, ["seed2"] = seed2 } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2596,6 +2713,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FractionalAvgPoolGrad", name) { args = new object[] { orig_input_tensor_shape, out_backprop, row_pooling_sequence, col_pooling_sequence }, attrs = new Dictionary() { ["overlapping"] = overlapping } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2731,6 +2852,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FractionalMaxPool", name) { args = new object[] { value }, attrs = new Dictionary() { ["pooling_ratio"] = pooling_ratio, ["pseudo_random"] = pseudo_random, ["overlapping"] = overlapping, ["deterministic"] = deterministic, ["seed"] = seed, ["seed2"] = seed2 } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2803,6 +2928,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FractionalMaxPoolGrad", name) { args = new object[] { orig_input, orig_output, out_backprop, row_pooling_sequence, col_pooling_sequence }, attrs = new Dictionary() { ["overlapping"] = overlapping } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2884,6 +3013,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNorm", name) { args = new object[] { x, scale, offset, mean, variance }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["exponential_avg_factor"] = exponential_avg_factor, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -2972,6 +3105,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNormGrad", name) { args = new object[] { y_backprop, x, scale, reserve_space_1, reserve_space_2 }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3059,6 +3196,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNormGradV2", name) { args = new object[] { y_backprop, x, scale, reserve_space_1, reserve_space_2 }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3147,6 +3288,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNormGradV3", name) { args = new object[] { y_backprop, x, scale, reserve_space_1, reserve_space_2, reserve_space_3 }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3235,6 +3380,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNormV2", name) { args = new object[] { x, scale, offset, mean, variance }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["exponential_avg_factor"] = exponential_avg_factor, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3323,6 +3472,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedBatchNormV3", name) { args = new object[] { x, scale, offset, mean, variance }, attrs = new Dictionary() { ["epsilon"] = epsilon, ["exponential_avg_factor"] = exponential_avg_factor, ["data_format"] = data_format, ["is_training"] = is_training } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3413,6 +3566,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedPadConv2D", name) { args = new object[] { input, paddings, filter }, attrs = new Dictionary() { ["mode"] = mode, ["strides"] = strides, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3502,6 +3659,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "FusedResizeAndPadConv2D", name) { args = new object[] { input, size, paddings, filter }, attrs = new Dictionary() { ["resize_align_corners"] = resize_align_corners, ["mode"] = mode, ["strides"] = strides, ["padding"] = padding } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3582,6 +3743,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InTopK", name) { args = new object[] { predictions, targets }, attrs = new Dictionary() { ["k"] = k } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3653,6 +3818,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "InTopKV2", name) { args = new object[] { predictions, targets, k }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3707,6 +3876,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "IsotonicRegression", name) { args = new object[] { input }, attrs = new Dictionary() { ["output_dtype"] = output_dtype } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3792,6 +3965,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LRN", name) { args = new object[] { input }, attrs = new Dictionary() { ["depth_radius"] = depth_radius, ["bias"] = bias, ["alpha"] = alpha, ["beta"] = beta } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3846,6 +4023,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LeakyRelu", name) { args = new object[] { features }, attrs = new Dictionary() { ["alpha"] = alpha } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3898,6 +4079,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LeakyReluGrad", name) { args = new object[] { gradients, features }, attrs = new Dictionary() { ["alpha"] = alpha } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -3956,6 +4141,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "LogSoftmax", name) { args = new object[] { logits }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4035,6 +4224,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPool", name) { args = new object[] { input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4119,6 +4312,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPool3D", name) { args = new object[] { input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4204,6 +4401,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPool3DGrad", name) { args = new object[] { orig_input, orig_output, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4291,6 +4492,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPool3DGradGrad", name) { args = new object[] { orig_input, orig_output, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4382,6 +4587,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGrad", name) { args = new object[] { orig_input, orig_output, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["explicit_paddings"] = explicit_paddings, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4469,6 +4678,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGradGrad", name) { args = new object[] { orig_input, orig_output, grad }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4546,6 +4759,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGradGradV2", name) { args = new object[] { orig_input, orig_output, grad, ksize, strides }, attrs = new Dictionary() { ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4628,6 +4845,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGradGradWithArgmax", name) { args = new object[] { input, grad, argmax }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["include_batch_in_index"] = include_batch_in_index } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4701,6 +4922,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGradV2", name) { args = new object[] { orig_input, orig_output, grad, ksize, strides }, attrs = new Dictionary() { ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4783,6 +5008,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolGradWithArgmax", name) { args = new object[] { input, grad, argmax }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding, ["include_batch_in_index"] = include_batch_in_index } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4854,6 +5083,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolV2", name) { args = new object[] { input, ksize, strides }, attrs = new Dictionary() { ["padding"] = padding, ["data_format"] = data_format } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -4946,6 +5179,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MaxPoolWithArgmax", name) { args = new object[] { input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["Targmax"] = Targmax, ["padding"] = padding, ["include_batch_in_index"] = include_batch_in_index } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5018,6 +5255,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "NthElement", name) { args = new object[] { input, n }, attrs = new Dictionary() { ["reverse"] = reverse } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5088,6 +5329,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedAvgPool", name) { args = new object[] { input, min_input, max_input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5174,6 +5419,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedBatchNormWithGlobalNormalization", name) { args = new object[] { t, t_min, t_max, m, m_min, m_max, v, v_min, v_max, beta, beta_min, beta_max, gamma, gamma_min, gamma_max }, attrs = new Dictionary() { ["out_type"] = out_type, ["variance_epsilon"] = variance_epsilon, ["scale_after_normalization"] = scale_after_normalization } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5251,6 +5500,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedBiasAdd", name) { args = new object[] { input, bias, min_input, max_input, min_bias, max_bias }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5344,6 +5597,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2D", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5420,6 +5677,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DAndRelu", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5499,6 +5760,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DAndReluAndRequantize", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5580,6 +5845,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DAndRequantize", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5662,6 +5931,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DPerChannel", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5739,6 +6012,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBias", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5818,6 +6095,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasAndRelu", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5899,6 +6180,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasAndReluAndRequantize", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -5982,6 +6267,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasAndRequantize", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6068,6 +6357,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasSignedSumAndReluAndRequantize", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output, summand, min_summand, max_summand }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6153,6 +6446,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasSumAndRelu", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, summand }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6238,6 +6535,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedConv2DWithBiasSumAndReluAndRequantize", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output, summand, min_summand, max_summand }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6322,6 +6623,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedDepthwiseConv2D", name) { args = new object[] { input, filter, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6400,6 +6705,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedDepthwiseConv2DWithBias", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6484,6 +6793,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedDepthwiseConv2DWithBiasAndRelu", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6571,6 +6884,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedDepthwiseConv2DWithBiasAndReluAndRequantize", name) { args = new object[] { input, filter, bias, min_input, max_input, min_filter, max_filter, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["out_type"] = out_type, ["strides"] = strides, ["padding"] = padding, ["dilations"] = dilations, ["padding_list"] = padding_list } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6660,6 +6977,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMulWithBias", name) { args = new object[] { a, b, bias, min_a, max_a, min_b, max_b }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["input_quant_mode"] = input_quant_mode } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6735,6 +7056,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMulWithBiasAndDequantize", name) { args = new object[] { a, b, bias, min_a, max_a, min_b, max_b, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["input_quant_mode"] = input_quant_mode } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6828,6 +7153,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMulWithBiasAndRelu", name) { args = new object[] { a, b, bias, min_a, max_a, min_b, max_b }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["input_quant_mode"] = input_quant_mode } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6922,6 +7251,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMulWithBiasAndReluAndRequantize", name) { args = new object[] { a, b, bias, min_a, max_a, min_b, max_b, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["input_quant_mode"] = input_quant_mode } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -6999,6 +7332,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMatMulWithBiasAndRequantize", name) { args = new object[] { a, b, bias, min_a, max_a, min_b, max_b, min_freezed_output, max_freezed_output }, attrs = new Dictionary() { ["Toutput"] = Toutput, ["transpose_a"] = transpose_a, ["transpose_b"] = transpose_b, ["input_quant_mode"] = input_quant_mode } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7083,6 +7420,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedMaxPool", name) { args = new object[] { input, min_input, max_input }, attrs = new Dictionary() { ["ksize"] = ksize, ["strides"] = strides, ["padding"] = padding } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7140,6 +7481,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedRelu", name) { args = new object[] { features, min_features, max_features }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7195,6 +7540,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedRelu6", name) { args = new object[] { features, min_features, max_features }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7251,6 +7600,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "QuantizedReluX", name) { args = new object[] { features, max_value, min_features, max_features }, attrs = new Dictionary() { ["out_type"] = out_type } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7312,6 +7665,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Relu", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7361,6 +7718,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Relu6", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7411,6 +7772,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReluGrad", name) { args = new object[] { gradients, features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7472,6 +7837,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Selu", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7522,6 +7891,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SeluGrad", name) { args = new object[] { gradients, outputs }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7579,6 +7952,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Softmax", name) { args = new object[] { logits }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7634,6 +8011,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SoftmaxCrossEntropyWithLogits", name) { args = new object[] { features, labels }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7684,6 +8065,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Softplus", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7734,6 +8119,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SoftplusGrad", name) { args = new object[] { gradients, features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7784,6 +8173,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "Softsign", name) { args = new object[] { features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7834,6 +8227,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SoftsignGrad", name) { args = new object[] { gradients, features }, attrs = new Dictionary() { } }); return _fast_path_result[0]; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7895,6 +8292,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "SparseSoftmaxCrossEntropyWithLogits", name) { args = new object[] { features, labels }, attrs = new Dictionary() { } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -7973,6 +8374,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TopK", name) { args = new object[] { input }, attrs = new Dictionary() { ["k"] = k, ["sorted"] = sorted } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } @@ -8045,6 +8450,10 @@ public static class gen_nn_ops var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "TopKV2", name) { args = new object[] { input, k }, attrs = new Dictionary() { ["sorted"] = sorted } }); return _fast_path_result; } + catch (NotOkStatusException ex) + { + throw ex; + } catch (Exception) { } diff --git a/tools/Tensorflow.CodeGen/GenOpsWriter.cs b/tools/Tensorflow.CodeGen/GenOpsWriter.cs index 7601acdb..9eefca07 100644 --- a/tools/Tensorflow.CodeGen/GenOpsWriter.cs +++ b/tools/Tensorflow.CodeGen/GenOpsWriter.cs @@ -39,6 +39,7 @@ namespace Tensorflow.CodeGen // Add commonly used namespaces. sb.AppendLine("using Tensorflow.Eager;"); sb.AppendLine("using Tensorflow.Contexts;"); + sb.AppendLine("using Tensorflow.Exceptions;"); sb.AppendLine("using static Tensorflow.Binding;"); sb.AppendLine(); From 675b93a9d752b300313c007069518dc75bf9784a Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Sat, 17 Jun 2023 23:10:37 +0800 Subject: [PATCH 13/77] fix: none gradient error when training LSTM. --- src/TensorFlowNET.Core/APIs/tf.tensor.cs | 6 +- src/TensorFlowNET.Core/Common/Types/Nest.cs | 18 +- .../Eager/EagerRunner.TFE_FastPathExecute.cs | 6 +- .../Eager/EagerRunner.TFE_TapeGradient.cs | 8 +- .../Gradients/array_grad.cs | 5 +- .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 2 - .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 26 +- .../ArgsDefinition/Rnn/StackedRNNCellsArgs.cs | 3 +- .../Keras/Layers/ILayersApi.cs | 2 +- .../Operations/NnOps/BasicLSTMCell.cs | 2 +- .../Operations/OpDefLibrary.cs | 9 +- .../Operations/_GraphTensorArray.cs | 3 +- .../Operations/array_ops.cs | 33 +- .../Operations/gen_resource_variable_ops.cs | 1573 +++++++++++++++-- .../Operations/image_ops_impl.cs | 6 +- src/TensorFlowNET.Core/Operations/while_v2.cs | 4 +- .../Variables/BaseResourceVariable.cs | 23 +- src/TensorFlowNET.Keras/Engine/Layer.Apply.cs | 2 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 15 +- src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 102 +- .../Layers/Rnn/LSTMCell.cs | 17 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 54 +- .../Layers/Rnn/SimpleRNN.cs | 7 +- .../Layers/Rnn/SimpleRNNCell.cs | 5 - .../Layers/Rnn/StackedRNNCells.cs | 12 +- .../Layers/Rnn.Test.cs | 74 +- tools/Tensorflow.CodeGen/OpClassifier.cs | 2 +- tools/Tensorflow.CodeGen/Utils.cs | 17 +- 29 files changed, 1743 insertions(+), 295 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.tensor.cs b/src/TensorFlowNET.Core/APIs/tf.tensor.cs index be8c2ab2..45aebc0c 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tensor.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tensor.cs @@ -71,15 +71,15 @@ namespace Tensorflow public Tensor[] split(Tensor value, int num_split, Tensor axis, string name = null) => array_ops.split( value: value, - num_split: num_split, + num_or_size_splits: num_split, axis: axis, name: name); public Tensor[] split(Tensor value, int num_split, int axis, string name = null) => array_ops.split( value: value, - num_split: num_split, - axis: axis, + num_or_size_splits: num_split, + axis: ops.convert_to_tensor(axis), name: name); public Tensor ensure_shape(Tensor x, Shape shape, string name = null) diff --git a/src/TensorFlowNET.Core/Common/Types/Nest.cs b/src/TensorFlowNET.Core/Common/Types/Nest.cs index 4de7d1fa..89ce29f2 100644 --- a/src/TensorFlowNET.Core/Common/Types/Nest.cs +++ b/src/TensorFlowNET.Core/Common/Types/Nest.cs @@ -197,25 +197,11 @@ namespace Tensorflow.Common.Types } else if(NestType is NestType.List) { - foreach(var item in ListValue!) - { - if(item.NestType is NestType.List or NestType.Dictionary) - { - return true; - } - } - return false; + return ListValue!.Count > 0; } else { - foreach (var item in DictValue!.Values) - { - if (item.NestType is NestType.List or NestType.Dictionary) - { - return true; - } - } - return false; + return DictValue!.Count > 0; } } diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs index 5f156fd9..0ce55841 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs @@ -352,7 +352,11 @@ namespace Tensorflow.Eager c_api.TFE_OpSetAttrFloat(op, key, Convert.ToSingle(value)); break; case TF_AttrType.TF_ATTR_SHAPE: - var dims = (value as long[]).ToArray(); + long[] dims; + if (value is Shape shape) dims = shape.dims.ToArray(); + else if (value is long[] longs) dims = longs.ToArray(); + else if (value is int[] ints) dims = ints.Select(x => (long)x).ToArray(); + else dims = ((long[])value).ToArray(); c_api.TFE_OpSetAttrShape(op, key, dims, dims.Length, status); status.Check(true); break; diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs index 1f7b3ae6..849dcb3f 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs @@ -137,7 +137,6 @@ namespace Tensorflow.Eager { dims[i] = c_api.TFE_TensorHandleDim(handle, i, status); } - Shape tensor_shape = new(dims); if(status.Code != TF_Code.TF_OK) { @@ -145,6 +144,7 @@ namespace Tensorflow.Eager } else { + Shape tensor_shape = new(dims); return new TapeTensor(id, dtype, tensor_shape); } } @@ -173,8 +173,12 @@ namespace Tensorflow.Eager return dtype == dtypes.variant || dtype == dtypes.resource; } - bool ListContainNone(long[] list) + bool ListContainNone(long[]? list) { + if(list is null) + { + return true; + } int len = list.Length; if(len == 0) { diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index f939f7b6..1b6bc95e 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -90,8 +90,7 @@ namespace Tensorflow.Gradients ? input_values[0].rank + dim_int : dim_int % input_values[0].rank; var sizes = input_values.Select(x => x.shape[non_neg_concat_dim]).ToArray(); - var sizes_tensor = constant_op.constant(sizes); - out_grads = array_ops.split(grad, sizes_tensor, non_neg_concat_dim).ToList(); + out_grads = array_ops.split(grad, sizes.Select(x => (int)x).ToArray(), ops.convert_to_tensor(non_neg_concat_dim)).ToList(); } else if (constant_op.is_constant(concat_dim)) { @@ -127,7 +126,7 @@ namespace Tensorflow.Gradients new Tensor[] { non_neg_concat_dim, tf.constant(0) }, new Tensor[] { tf.constant(1), tf.constant(-1) }); var squeeze_sizes = array_ops.squeeze(slice); - out_grads = array_ops.split(axis: grad, value: squeeze_sizes, num_split: (int)non_neg_concat_dim).ToList(); + out_grads = array_ops.split(axis: grad, value: squeeze_sizes, num_or_size_splits: (int)non_neg_concat_dim).ToList(); } else { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index 76464147..db76fda0 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -4,8 +4,6 @@ { // TODO: maybe change the `RNNArgs` and implement this class. public bool UnitForgetBias { get; set; } - public float Dropout { get; set; } - public float RecurrentDropout { get; set; } public int Implementation { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 786236e4..1b26c05c 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -29,7 +29,7 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn [JsonProperty("unit_forget_bias")] public bool UnitForgetBias { get; set; } = true; [JsonProperty("implementation")] - public int Implementation { get; set; } = 2; + public int Implementation { get; set; } = 1; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 116ff7a2..2d7fb001 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -7,12 +7,6 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs { - [JsonProperty("cell")] - // TODO: the cell should be serialized with `serialize_keras_object`. - public IRnnCell Cell { get; set; } = null; - [JsonProperty("cells")] - public IList Cells { get; set; } = null; - [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; [JsonProperty("return_state")] @@ -25,8 +19,10 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public bool Unroll { get; set; } = false; [JsonProperty("time_major")] public bool TimeMajor { get; set; } = false; + + public int? InputDim { get; set; } + public int? InputLength { get; set; } // TODO: Add `num_constants` and `zero_output_for_mask`. - public Dictionary Kwargs { get; set; } = null; public int Units { get; set; } public Activation Activation { get; set; } @@ -38,21 +34,5 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public float Dropout { get; set; } = .0f; public bool ZeroOutputForMask { get; set; } = false; public float RecurrentDropout { get; set; } = .0f; - - // kernel_regularizer=None, - // recurrent_regularizer=None, - // bias_regularizer=None, - // activity_regularizer=None, - // kernel_constraint=None, - // recurrent_constraint=None, - // bias_constraint=None, - // dropout=0., - // recurrent_dropout=0., - // return_sequences=False, - // return_state=False, - // go_backwards=False, - // stateful=False, - // unroll=False, - // **kwargs): } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index ea6f830b..50a6127d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -5,7 +5,6 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class StackedRNNCellsArgs : LayerArgs { - public IList Cells { get; set; } - public Dictionary Kwargs { get; set; } = null; + public bool ReverseStateOrder = false; } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index a19508d4..1eb08e77 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -182,7 +182,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2, + int implementation = 1, bool return_sequences = false, bool return_state = false, bool go_backwards = false, diff --git a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs index b2cda952..16cbd001 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/BasicLSTMCell.cs @@ -89,7 +89,7 @@ namespace Tensorflow gate_inputs = nn_ops.bias_add(gate_inputs, _bias); // i = input_gate, j = new_input, f = forget_gate, o = output_gate - var tensors = array_ops.split(value: gate_inputs, num_split: 4, axis: one); + var tensors = array_ops.split(value: gate_inputs, num_or_size_splits: 4, axis: one); var (i, j, f, o) = (tensors[0], tensors[1], tensors[2], tensors[3]); var forget_bias_tensor = constant_op.constant(_forget_bias, dtype: f.dtype); diff --git a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs index 5ff5ccff..29e1f074 100644 --- a/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs +++ b/src/TensorFlowNET.Core/Operations/OpDefLibrary.cs @@ -389,9 +389,13 @@ namespace Tensorflow case "list(type)": attr_value.List.Type.AddRange((value as IList).Select(x => _MakeType(x, attr_def))); break; + case "list(float)": + if (value != null) + attr_value.List.F.AddRange((value as IEnumerable).ToArray()); + break; case "list(int)": if (value != null) - attr_value.List.I.AddRange((value as int[]).Select(x => Convert.ToInt64(x))); + attr_value.List.I.AddRange((value as IEnumerable).Select(x => Convert.ToInt64(x))); break; case "bool": attr_value.B = (bool)value; @@ -428,6 +432,9 @@ namespace Tensorflow case "list(func)": attr_value.List.Func.AddRange(_MakeFuncList(value, attr_def.Name)); break; + case "list(string)": + attr_value.List.S.AddRange((value as IEnumerable).Select(x => ByteString.CopyFromUtf8(x))); + break; default: throw new TypeError($"SetAttrValue: can't not convert attr_def.Type '{attr_def.Type}' to protos."); } diff --git a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs index 4c3fde31..2384e814 100644 --- a/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs +++ b/src/TensorFlowNET.Core/Operations/_GraphTensorArray.cs @@ -390,7 +390,8 @@ namespace Tensorflow.Operations int ta_size; if(!_dynamic_size && (_size is not null)) { - ta_size = (int)tensor_util.constant_value(_size); + var size_tensor = tensor_util.constant_value(_size); + ta_size = size_tensor is null ? -1 : (int)size_tensor; } else { diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index c4ec974b..6b4fea63 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -1014,38 +1014,27 @@ namespace Tensorflow }); } - public static Tensor[] split(Tensor value, Tensor size_splits, int axis, int num = -1, + public static Tensor[] split(Tensor value, int num_or_size_splits, Tensor axis = null, string name = "split") { - if (num == -1) - num = (int)size_splits.shape[0]; - - return gen_array_ops.split_v(value, size_splits, tf.convert_to_tensor(axis), num, name: name); + return gen_array_ops.split(split_dim: axis, value: value, num_split: num_or_size_splits, name); } - public static Tensor[] split(Tensor value, int num_split, T axis, + public static Tensor[] split(Tensor value, int[] num_or_size_splits, Tensor axis = null, int num = -1, string name = "split") { - var size_splits = ops.convert_to_tensor(num_split); - - if (tf.Context.executing_eagerly()) + if(num_or_size_splits.Length == 0) { - return split_eager_fallback(axis, value, num_split: num_split, name: name, ctx: tf.Context); + throw new ValueError("Rank-0 tensors are not supported as the num_or_size_splits argument to split."); } + var size_splits = ops.convert_to_tensor(num_or_size_splits); - var _op = tf.OpDefLib._apply_op_helper("Split", name, new { split_dim = axis, value, num_split }); - return _op.outputs; - } - - private static Tensor[] split_eager_fallback(Ta axis, Tv value, int num_split, string name, Context ctx = null) - { - var (_attr_T, input) = tf.Runner.ArgsToMatchingEager(ctx, args: new object[] { value }); - var axis_tensor = ops.convert_to_tensor(axis, dtype: TF_DataType.TF_INT32); - var _inputs_flat = new List { axis_tensor }; - _inputs_flat.AddRange(input); - var _attrs = new object[] { "num_split", num_split, "T", _attr_T }; + if(num == -1) + { + num = (int)size_splits.shape[0]; + } - return tf.Runner.Execute(ctx, "Split", num_split, _inputs_flat.ToArray(), _attrs, name: name); + return gen_array_ops.split_v(value: value, size_splits: size_splits, split_dim: axis, num_split: num, name: name); } public static Tensor slice(Tensor input, Tensor[] begin, Tensor[] size, string name = null) diff --git a/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs b/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs index c4e8f8c4..db5f6813 100644 --- a/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_resource_variable_ops.cs @@ -1,158 +1,1523 @@ -/***************************************************************************** - Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ +/*Wrappers around TensorFlow ops. This file is MACHINE GENERATED! Do not edit.*/ +using Tensorflow.Eager; +using Tensorflow.Contexts; +using Tensorflow.Exceptions; using static Tensorflow.Binding; -namespace Tensorflow +namespace Tensorflow; + +public static class gen_resource_variable_ops { - public static class gen_resource_variable_ops + /// + /// Adds a value to the current value of a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to + /// see the incremented value or a subsequent newer one. + /// + /// + /// + /// + /// + public static Operation assign_add_variable_op(Tensor resource, Tensor value, string? name = null) { - public static Operation assign_sub_variable_op(Tensor resource, Tensor value, string name = null) + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - if (tf.Context.executing_eagerly()) + try { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo( - tf.Context, "AssignSubVariableOp", name, resource, value)); - + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignAddVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return assign_add_variable_op_eager_fallback(resource, value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + var _op = tf.OpDefLib._apply_op_helper("AssignAddVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("AssignAddVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } - return null; + public static Operation assign_add_variable_op_eager_fallback(Tensor resource, Tensor value, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype }; + var _result = _execute.execute("AssignAddVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignAddVariableOp", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Subtracts a value from the current value of a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to + /// see the decremented value or a subsequent newer one. + /// + /// + /// + /// + /// + public static Operation assign_sub_variable_op(Tensor resource, Tensor value, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignSubVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return assign_sub_variable_op_eager_fallback(resource, value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + var _op = tf.OpDefLib._apply_op_helper("AssignSubVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("AssignSubVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } - /// - /// Adds a value to the current value of a variable. - /// - /// - /// - /// - /// - public static Operation assign_add_variable_op(Tensor resource, Tensor value, string name = null) + public static Operation assign_sub_variable_op_eager_fallback(Tensor resource, Tensor value, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype }; + var _result = _execute.execute("AssignSubVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignSubVariableOp", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Assigns a new value to a variable. + /// + /// + /// + /// Any ReadVariableOp with a control dependency on this op is guaranteed to return + /// this value or a subsequent newer value of the variable. + /// + /// + /// + /// + /// + /// + public static Operation assign_variable_op(Tensor resource, Tensor value, bool validate_shape = false, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) { - if (tf.Context.executing_eagerly()) + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AssignVariableOp", name) { args = new object[] { resource, value }, attrs = new Dictionary() { ["validate_shape"] = validate_shape } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "AssignAddVariableOp", name, - resource, value)); + } + try + { + return assign_variable_op_eager_fallback(resource, value, validate_shape: validate_shape, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["value"] = value; + keywords["validate_shape"] = validate_shape; + var _op = tf.OpDefLib._apply_op_helper("AssignVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "validate_shape", _op._get_attr_bool("validate_shape") }; + _execute.record_gradient("AssignVariableOp", _op.inputs, _attrs, _result); + } + return _op; + } + public static Operation assign_variable_op_eager_fallback(Tensor resource, Tensor value, bool validate_shape, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, value }; + object[] _attrs = new object[] { "dtype", value.dtype, "validate_shape", validate_shape }; + var _result = _execute.execute("AssignVariableOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AssignVariableOp", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// This op consumes a lock created by `MutexLock`. + /// + /// + /// + /// This op exists to consume a tensor created by `MutexLock` (other than + /// direct control dependencies). It should be the only that consumes the tensor, + /// and will raise an error if it is not. Its only purpose is to keep the + /// mutex lock tensor alive until it is consumed by this op. + /// + /// **NOTE**: This operation must run on the same device as its input. This may + /// be enforced via the `colocate_with` mechanism. + /// + /// + /// + /// + public static Operation consume_mutex_lock(Tensor mutex_lock, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ConsumeMutexLock", name) { args = new object[] { mutex_lock }, attrs = new Dictionary() { } }); return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return consume_mutex_lock_eager_fallback(mutex_lock, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["mutex_lock"] = mutex_lock; + var _op = tf.OpDefLib._apply_op_helper("ConsumeMutexLock", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("ConsumeMutexLock", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("AssignAddVariableOp", name, new { resource, value }); + public static Operation consume_mutex_lock_eager_fallback(Tensor mutex_lock, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { mutex_lock }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("ConsumeMutexLock", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ConsumeMutexLock", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Deletes the resource specified by the handle. + /// + /// + /// + /// All subsequent operations using the resource will result in a NotFound + /// error status. + /// + /// + /// + /// + /// + /// whether to ignore the error when the resource + /// doesn't exist. + /// + /// + /// + public static Operation destroy_resource_op(Tensor resource, bool ignore_lookup_error = true, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DestroyResourceOp", name) { args = new object[] { resource }, attrs = new Dictionary() { ["ignore_lookup_error"] = ignore_lookup_error } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return destroy_resource_op_eager_fallback(resource, ignore_lookup_error: ignore_lookup_error, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["ignore_lookup_error"] = ignore_lookup_error; + var _op = tf.OpDefLib._apply_op_helper("DestroyResourceOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "ignore_lookup_error", _op._get_attr_bool("ignore_lookup_error") }; + _execute.record_gradient("DestroyResourceOp", _op.inputs, _attrs, _result); + } + return _op; + } - return _op; + public static Operation destroy_resource_op_eager_fallback(Tensor resource, bool ignore_lookup_error, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { "ignore_lookup_error", ignore_lookup_error }; + var _result = _execute.execute("DestroyResourceOp", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("DestroyResourceOp", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Turns off the copy-on-read mode. + /// + /// + /// + /// Turns off the copy-on-read mode of a resource variable. If the variable is not in copy-on-read mode, this op has no effect. + /// + /// + /// + /// + public static Operation disable_copy_on_read(Tensor resource, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "DisableCopyOnRead", name) { args = new object[] { resource }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return disable_copy_on_read_eager_fallback(resource, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + var _op = tf.OpDefLib._apply_op_helper("DisableCopyOnRead", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("DisableCopyOnRead", _op.inputs, _attrs, _result); + } + return _op; + } - public static Operation assign_variable_op(Tensor resource, Tensor value, string name = null) + public static Operation disable_copy_on_read_eager_fallback(Tensor resource, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("DisableCopyOnRead", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("DisableCopyOnRead", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Locks a mutex resource. The output is the lock. So long as the lock tensor + /// + /// + /// + /// is alive, any other request to use `MutexLock` with this mutex will wait. + /// + /// This is particularly useful for creating a critical section when used in + /// conjunction with `MutexLockIdentity`: + /// + /// ```python + /// + /// mutex = mutex_v2( + /// shared_name=handle_name, container=container, name=name) + /// + /// def execute_in_critical_section(fn, *args, **kwargs): + /// lock = gen_resource_variable_ops.mutex_lock(mutex) + /// + /// with ops.control_dependencies([lock]): + /// r = fn(*args, **kwargs) + /// + /// with ops.control_dependencies(nest.flatten(r)): + /// with ops.colocate_with(mutex): + /// ensure_lock_exists = mutex_lock_identity(lock) + /// + /// # Make sure that if any element of r is accessed, all of + /// # them are executed together. + /// r = nest.map_structure(tf.identity, r) + /// + /// with ops.control_dependencies([ensure_lock_exists]): + /// return nest.map_structure(tf.identity, r) + /// ``` + /// + /// While `fn` is running in the critical section, no other functions which wish to + /// use this critical section may run. + /// + /// Often the use case is that two executions of the same graph, in parallel, + /// wish to run `fn`; and we wish to ensure that only one of them executes + /// at a time. This is especially important if `fn` modifies one or more + /// variables at a time. + /// + /// It is also useful if two separate functions must share a resource, but we + /// wish to ensure the usage is exclusive. + /// + /// + /// + /// + public static Tensor mutex_lock(Tensor mutex, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MutexLock", name) { args = new object[] { mutex }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try { - tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "AssignVariableOp", name, - resource, value)); + return mutex_lock_eager_fallback(mutex, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["mutex"] = mutex; + var _op = tf.OpDefLib._apply_op_helper("MutexLock", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("MutexLock", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return null; + public static Tensor mutex_lock_eager_fallback(Tensor mutex, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { mutex }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("MutexLock", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("MutexLock", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Creates a Mutex resource that can be locked by `MutexLock`. + /// + /// + /// + /// If non-empty, this variable is placed in the given container. + /// Otherwise, a default container is used. + /// + /// + /// + /// + /// If non-empty, this variable is named in the given bucket + /// with this shared_name. Otherwise, the node name is used instead. + /// + /// + /// + public static Tensor mutex_v2(string container = "", string shared_name = "", string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "MutexV2", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { } + try + { + return mutex_v2_eager_fallback(container: container, shared_name: shared_name, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } + Dictionary keywords = new(); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + var _op = tf.OpDefLib._apply_op_helper("MutexV2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "container", _op.get_attr("container"), "shared_name", _op.get_attr("shared_name") }; + _execute.record_gradient("MutexV2", _op.inputs, _attrs, _result); + } + return _result[0]; + } - var _op = tf.OpDefLib._apply_op_helper("AssignVariableOp", name, new { resource, value }); + public static Tensor mutex_v2_eager_fallback(string container, string shared_name, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "container", container, "shared_name", shared_name }; + var _result = _execute.execute("MutexV2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("MutexV2", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Reads the value of a variable. + /// + /// + /// + /// The tensor returned by this operation is immutable. + /// + /// The value returned by this operation is guaranteed to be influenced by all the + /// writes on which this operation depends directly or indirectly, and to not be + /// influenced by any of the writes which depend directly or indirectly on this + /// operation. + /// + /// + /// + /// + /// + /// the dtype of the value. + /// + /// + /// + public static Tensor read_variable_op(Tensor resource, TF_DataType dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ReadVariableOp", name) { args = new object[] { resource }, attrs = new Dictionary() { ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return read_variable_op_eager_fallback(resource, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ReadVariableOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype") }; + _execute.record_gradient("ReadVariableOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return _op; + public static Tensor read_variable_op_eager_fallback(Tensor resource, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { "dtype", dtype }; + var _result = _execute.execute("ReadVariableOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ReadVariableOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Gather slices from the variable pointed to by `resource` according to `indices`. + /// + /// + /// + /// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D). + /// Produces an output tensor with shape `indices.shape + params.shape[1:]` where: + /// + /// ```python + /// # Scalar indices + /// output[:, ..., :] = params[indices, :, ... :] + /// + /// # Vector indices + /// output[i, :, ..., :] = params[indices[i], :, ... :] + /// + /// # Higher rank indices + /// output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :] + /// ``` + /// + /// + /// + /// + /// + /// + /// + /// + public static Tensor resource_gather(Tensor resource, Tensor indices, TF_DataType dtype, int batch_dims = 0, bool validate_indices = true, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceGather", name) { args = new object[] { resource, indices }, attrs = new Dictionary() { ["batch_dims"] = batch_dims, ["validate_indices"] = validate_indices, ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_gather_eager_fallback(resource, indices, batch_dims: batch_dims, validate_indices: validate_indices, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["batch_dims"] = batch_dims; + keywords["validate_indices"] = validate_indices; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ResourceGather", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "batch_dims", _op._get_attr_int("batch_dims"), "validate_indices", _op._get_attr_bool("validate_indices"), "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceGather", _op.inputs, _attrs, _result); } + return _result[0]; + } - public static Tensor var_is_initialized_op(Tensor resource, string name = null) + public static Tensor resource_gather_eager_fallback(Tensor resource, Tensor indices, int batch_dims, bool validate_indices, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices }; + object[] _attrs = new object[] { "batch_dims", batch_dims, "validate_indices", validate_indices, "dtype", dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceGather", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("ResourceGather", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// + /// + /// + /// + /// + /// + public static Tensor resource_gather_nd(Tensor resource, Tensor indices, TF_DataType dtype, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try { - var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "VarIsInitializedOp", name, - resource)); + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceGatherNd", name) { args = new object[] { resource, indices }, attrs = new Dictionary() { ["dtype"] = dtype } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_gather_nd_eager_fallback(resource, indices, dtype: dtype, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["dtype"] = dtype; + var _op = tf.OpDefLib._apply_op_helper("ResourceGatherNd", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceGatherNd", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return results[0]; + public static Tensor resource_gather_nd_eager_fallback(Tensor resource, Tensor indices, TF_DataType dtype, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices }; + object[] _attrs = new object[] { "dtype", dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceGatherNd", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceGatherNd", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Adds sparse updates to the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] += updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] += updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] += updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions add. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_add(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterAdd", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; } + catch (Exception) + { + } + try + { + return resource_scatter_add_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterAdd", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterAdd", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("VarIsInitializedOp", name, new { resource }); + public static Operation resource_scatter_add_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterAdd", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterAdd", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Divides sparse updates into the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] /= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] /= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] /= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions multiply. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_div(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterDiv", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_div_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterDiv", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterDiv", _op.inputs, _attrs, _result); + } + return _op; + } - return _op.output; + public static Operation resource_scatter_div_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterDiv", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterDiv", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Reduces sparse updates into the variable referenced by `resource` using the `max` operation. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = max(ref[indices, ...], updates[...]) + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...]) + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...]) + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions are combined. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_max(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMax", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_max_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMax", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMax", _op.inputs, _attrs, _result); + } + return _op; + } - /// - /// Creates a handle to a Variable resource. - /// - /// - /// - /// - /// - /// - /// - public static Tensor var_handle_op(TF_DataType dtype, Shape shape, - string container = "", string shared_name = "", string name = null) + public static Operation resource_scatter_max_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMax", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - if (tf.Context.executing_eagerly()) + _execute.record_gradient("ResourceScatterMax", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Reduces sparse updates into the variable referenced by `resource` using the `min` operation. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = min(ref[indices, ...], updates[...]) + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = min(ref[indices[i], ...], updates[i, ...]) + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = min(ref[indices[i, ..., j], ...], updates[i, ..., j, ...]) + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions are combined. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_min(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMin", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_min_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) { - var results = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(tf.Context, "VarHandleOp", name) - { - attrs = ConvertToDict(new - { - dtype, - shape = shape.dims, - container, - shared_name, - allowed_devices = new string[0] - }) - }); + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMin", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMin", _op.inputs, _attrs, _result); + } + return _op; + } - return results[0]; + public static Operation resource_scatter_min_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMin", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterMin", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Multiplies sparse updates into the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] *= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] *= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions multiply. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_mul(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterMul", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_mul_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterMul", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterMul", _op.inputs, _attrs, _result); + } + return _op; + } - var _op = tf.OpDefLib._apply_op_helper("VarHandleOp", name, new + public static Operation resource_scatter_mul_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterMul", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterMul", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Subtracts sparse updates from the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] -= updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] -= updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...] + /// + /// Duplicate entries are handled correctly: if multiple `indices` reference + /// the same location, their contributions add. + /// + /// Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`. + /// + ///
+ /// + ///
+ /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_sub(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterSub", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_sub_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) { - dtype, - shape, - container, - shared_name - }); + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterSub", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterSub", _op.inputs, _attrs, _result); + } + return _op; + } - return _op.output; + public static Operation resource_scatter_sub_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterSub", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterSub", _inputs_flat, _attrs, _result); } + return null; + } + /// + /// Assigns sparse updates to the variable referenced by `resource`. + /// + /// + /// + /// This operation computes + /// + /// # Scalar indices + /// ref[indices, ...] = updates[...] + /// + /// # Vector indices (for each i) + /// ref[indices[i], ...] = updates[i, ...] + /// + /// # High rank indices (for each i, ..., j) + /// ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + /// + /// + /// + /// + /// + /// + public static Operation resource_scatter_update(Tensor resource, Tensor indices, Tensor updates, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "ResourceScatterUpdate", name) { args = new object[] { resource, indices, updates }, attrs = new Dictionary() { } }); + return null; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return resource_scatter_update_eager_fallback(resource, indices, updates, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + keywords["indices"] = indices; + keywords["updates"] = updates; + var _op = tf.OpDefLib._apply_op_helper("ResourceScatterUpdate", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "dtype", _op._get_attr_type("dtype"), "Tindices", _op._get_attr_type("Tindices") }; + _execute.record_gradient("ResourceScatterUpdate", _op.inputs, _attrs, _result); + } + return _op; + } - public static Tensor destroy_resource_op(Tensor resource, bool ignore_lookup_error = true, string name = null) - => tf.Context.ExecuteOp("DestroyResourceOp", name, - new ExecuteOpArgs(resource).SetAttributes(new { ignore_lookup_error })); + public static Operation resource_scatter_update_eager_fallback(Tensor resource, Tensor indices, Tensor updates, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource, indices, updates }; + object[] _attrs = new object[] { "dtype", updates.dtype, "Tindices", indices.dtype }; + var _result = _execute.execute("ResourceScatterUpdate", 0, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("ResourceScatterUpdate", _inputs_flat, _attrs, _result); + } + return null; + } + /// + /// Creates a handle to a Variable resource. + /// + /// + /// + /// the container this variable is placed in. + /// + /// + /// + /// + /// the name by which this variable is referred to. + /// + /// + /// + /// + /// the type of this variable. Must agree with the dtypes + /// of all ops using this variable. + /// + /// + /// + /// + /// The (possibly partially specified) shape of this variable. + /// + /// + /// + /// + /// DEPRECATED. The allowed devices containing the resource variable. Set when the + /// output ResourceHandle represents a per-replica/partitioned resource variable. + /// + /// + /// + public static Tensor var_handle_op(TF_DataType dtype, Shape shape, string container = "", string shared_name = "", string[] allowed_devices = null, string? name = null) + { + var _ctx = tf.Context; + if (allowed_devices is null) + { + allowed_devices = new string[] { }; + } + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VarHandleOp", name) { args = new object[] { }, attrs = new Dictionary() { ["container"] = container, ["shared_name"] = shared_name, ["dtype"] = dtype, ["shape"] = shape, ["allowed_devices"] = allowed_devices } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return var_handle_op_eager_fallback(container: container, shared_name: shared_name, dtype: dtype, shape: shape, allowed_devices: allowed_devices, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + if (container is null) + { + container = ""; + } + if (shared_name is null) + { + shared_name = ""; + } + Dictionary keywords = new(); + keywords["container"] = container; + keywords["shared_name"] = shared_name; + keywords["dtype"] = dtype; + keywords["shape"] = shape; + keywords["allowed_devices"] = allowed_devices; + var _op = tf.OpDefLib._apply_op_helper("VarHandleOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "container", _op.get_attr("container"), "shared_name", _op.get_attr("shared_name"), "dtype", _op._get_attr_type("dtype"), "shape", _op.get_attr("shape"), "allowed_devices", _op.get_attr("allowed_devices") }; + _execute.record_gradient("VarHandleOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - /// - /// Reads the value of a variable. - /// - /// - /// - /// - /// - public static Tensor read_variable_op(Tensor resource, TF_DataType dtype, string name = null) - => tf.Context.ExecuteOp("ReadVariableOp", name, new ExecuteOpArgs(resource) - .SetAttributes(new { dtype })); + public static Tensor var_handle_op_eager_fallback(string container, string shared_name, TF_DataType dtype, Shape shape, string[] allowed_devices, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { }; + object[] _attrs = new object[] { "container", container, "shared_name", shared_name, "dtype", dtype, "shape", shape, "allowed_devices", allowed_devices }; + var _result = _execute.execute("VarHandleOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("VarHandleOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Checks whether a resource handle-based variable has been initialized. + /// + /// + /// + public static Tensor var_is_initialized_op(Tensor resource, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VarIsInitializedOp", name) { args = new object[] { resource }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return var_is_initialized_op_eager_fallback(resource, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["resource"] = resource; + var _op = tf.OpDefLib._apply_op_helper("VarIsInitializedOp", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { }; + _execute.record_gradient("VarIsInitializedOp", _op.inputs, _attrs, _result); + } + return _result[0]; + } - public static Tensor resource_gather(Tensor resource, Tensor indices, TF_DataType dtype, - int batch_dims = 0, bool validate_indices = true, string name = null) + public static Tensor var_is_initialized_op_eager_fallback(Tensor resource, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { resource }; + object[] _attrs = new object[] { }; + var _result = _execute.execute("VarIsInitializedOp", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) { - var _op = tf.OpDefLib._apply_op_helper("ResourceGather", name, new + _execute.record_gradient("VarIsInitializedOp", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + /// + /// Returns the shape of the variable pointed to by `resource`. + /// + /// + /// + /// This operation returns a 1-D integer tensor representing the shape of `input`. + /// + /// For example: + /// + /// ``` + /// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] + /// shape(t) ==> [2, 2, 3] + /// ``` + /// + /// + /// + /// + /// + public static Tensor variable_shape(Tensor input, TF_DataType out_type = TF_DataType.TF_INT32, string? name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "VariableShape", name) { args = new object[] { input }, attrs = new Dictionary() { ["out_type"] = out_type } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) { - resource, - indices, - dtype, - batch_dims, - validate_indices - }); + throw ex; + } + catch (Exception) + { + } + try + { + return variable_shape_eager_fallback(input, out_type: out_type, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["input"] = input; + keywords["out_type"] = out_type; + var _op = tf.OpDefLib._apply_op_helper("VariableShape", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "out_type", _op._get_attr_type("out_type") }; + _execute.record_gradient("VariableShape", _op.inputs, _attrs, _result); + } + return _result[0]; + } - return _op.output; + public static Tensor variable_shape_eager_fallback(Tensor input, TF_DataType out_type, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { input }; + object[] _attrs = new object[] { "out_type", out_type }; + var _result = _execute.execute("VariableShape", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("VariableShape", _inputs_flat, _attrs, _result); } + return _result[0]; } } diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 9d52f516..126df9e4 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -1778,10 +1778,10 @@ new_height, new_width"); { // a_y_min: [0], a_x_min: [1], a_y_max: [2], a_x_max[3] var a_xy_minmax = array_ops.split( - value: boxes_a, num_split: 4, axis: 2); + value: boxes_a, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); // b_y_min: [0], b_x_min: [1], b_y_max: [2], b_x_max[3] var b_xy_minmax = array_ops.split( - value: boxes_b, num_split: 4, axis: 2); + value: boxes_b, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); var i_xmin = math_ops.maximum( a_xy_minmax[1], array_ops.transpose(b_xy_minmax[1], new[] { 0, 2, 1 })); @@ -1943,7 +1943,7 @@ new_height, new_width"); using (ops.name_scope("canonicalize_coordinates")) { // y_1 = [0], x_1 = [1], y_2 = [2], x_2 = [3] - var yx = array_ops.split(value: boxes, num_split: 4, axis: 2); + var yx = array_ops.split(value: boxes, num_or_size_splits: 4, axis: ops.convert_to_tensor(2)); var y_1_is_min = math_ops.reduce_all( gen_math_ops.less_equal(yx[0][0, 0, 0], yx[2][0, 0, 0])); var y_minmax = control_flow_ops.cond( diff --git a/src/TensorFlowNET.Core/Operations/while_v2.cs b/src/TensorFlowNET.Core/Operations/while_v2.cs index 7ee3e9e8..3f324f87 100644 --- a/src/TensorFlowNET.Core/Operations/while_v2.cs +++ b/src/TensorFlowNET.Core/Operations/while_v2.cs @@ -86,7 +86,7 @@ namespace Tensorflow.Operations } } - var cond_graph = FuncGraph.func_graph_from_func("cond", wrapped_cond, null, + var cond_graph = FuncGraph.func_graph_from_func(cond_name, wrapped_cond, null, null, signature: func_graph_signature, add_control_dependencies: add_control_dependencies); bool stateful_parallelism = false; @@ -111,7 +111,7 @@ namespace Tensorflow.Operations return new object[] { loop_counter + 1, maximum_iterations_arg }.Concat(outputs).ToArray(); } - var body_graph = FuncGraph.func_graph_from_func("body", wrapped_body, null, null, func_graph_signature, + var body_graph = FuncGraph.func_graph_from_func(body_name, wrapped_body, null, null, func_graph_signature, add_control_dependencies: add_control_dependencies, acd_record_initial_resource_uses: stateful_parallelism); // TODO(Rinne): possible wrong implementation here. diff --git a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs index b9a7022a..a54283bd 100644 --- a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs @@ -170,11 +170,28 @@ namespace Tensorflow public Tensor value() => GraphElement ?? _read_variable_op(); - protected Tensor _read_variable_op() + protected Tensor _read_variable_op(bool no_copy = false) { variable_accessed(this); - var result = gen_resource_variable_ops.read_variable_op(handle, _dtype); - resource_variable_ops._maybe_set_handle_data(_dtype, handle, result); + + Tensor read_and_set_handle(bool no_copy) + { + if (no_copy) + { + gen_resource_variable_ops.disable_copy_on_read(handle); + } + var result = gen_resource_variable_ops.read_variable_op(handle, _dtype); + resource_variable_ops._maybe_set_handle_data(_dtype, handle, result); + return result; + } + + // TODO(Rinne): deal with caching device. + var result = read_and_set_handle(no_copy); + if (!tf.Context.executing_eagerly()) + { + tf.Runner.TFE_TapeSetRecordOperation("ReadVariableOp", new Tensor[] { result }, new Tensor[] { handle }, + backward_function: (x, _) => x); + } // have to set shape when converting to substituent placeholder if (result.shape.ndim == -1) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index a0358f07..d52190fd 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -38,6 +38,8 @@ namespace Tensorflow.Keras.Engine _handle_activity_regularization(inputs, outputs); _set_mask_metadata(inputs, outputs, null); + // TODO(Rinne): set save spec if null + scope.__exit__(); return outputs; diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 66c3cdc1..efca9300 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -709,10 +709,7 @@ namespace Tensorflow.Keras.Layers public IRnnCell StackedRNNCells( IEnumerable cells) - => new StackedRNNCells(new StackedRNNCellsArgs - { - Cells = cells.ToList() - }); + => new StackedRNNCells(cells.ToList(), new StackedRNNCellsArgs()); /// /// @@ -757,9 +754,8 @@ namespace Tensorflow.Keras.Layers bool stateful = false, bool unroll = false, bool time_major = false) - => new RNN(new RNNArgs + => new RNN(cell, new RNNArgs { - Cell = cell, ReturnSequences = return_sequences, ReturnState = return_state, GoBackwards = go_backwards, @@ -776,9 +772,8 @@ namespace Tensorflow.Keras.Layers bool stateful = false, bool unroll = false, bool time_major = false) - => new RNN(new RNNArgs + => new RNN(cell, new RNNArgs { - Cells = cell.ToList(), ReturnSequences = return_sequences, ReturnState = return_state, GoBackwards = go_backwards, @@ -798,7 +793,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2) + int implementation = 1) => new LSTMCell(new LSTMCellArgs { Units = uints, @@ -851,7 +846,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 2, + int implementation = 1, bool return_sequences = false, bool return_state = false, bool go_backwards = false, diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 1449c908..025465fd 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -2,6 +2,7 @@ using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; +using Tensorflow.Common.Extensions; namespace Tensorflow.Keras.Layers.Rnn { @@ -14,22 +15,105 @@ namespace Tensorflow.Keras.Layers.Rnn public class LSTM : RNN { LSTMArgs args; - InputSpec[] state_spec; - - int units => args.Units; + InputSpec[] _state_spec; + InputSpec _input_spec; + bool _could_use_gpu_kernel; public LSTM(LSTMArgs args) : - base(args) + base(CreateCell(args), args) { this.args = args; - state_spec = new[] { units, units } - .Select(dim => new InputSpec(shape: (-1, dim))) - .ToArray(); + _input_spec = new InputSpec(ndim: 3); + _state_spec = new[] { args.Units, args.Units }.Select(dim => new InputSpec(shape: (-1, dim))).ToArray(); + _could_use_gpu_kernel = args.Activation == keras.activations.Tanh + && args.RecurrentActivation == keras.activations.Sigmoid + && args.RecurrentDropout == 0 && !args.Unroll && args.UseBias + && ops.executing_eagerly_outside_functions(); + } + + private static IRnnCell CreateCell(LSTMArgs lstmArgs) + { + return new LSTMCell(new LSTMCellArgs() + { + Units = lstmArgs.Units, + Activation = lstmArgs.Activation, + RecurrentActivation = lstmArgs.RecurrentActivation, + UseBias = lstmArgs.UseBias, + KernelInitializer = lstmArgs.KernelInitializer, + RecurrentInitializer = lstmArgs.RecurrentInitializer, + UnitForgetBias = lstmArgs.UnitForgetBias, + BiasInitializer = lstmArgs.BiasInitializer, + // TODO(Rinne): kernel_regularizer + // TODO(Rinne): recurrent_regularizer + // TODO(Rinne): bias_regularizer + // TODO(Rinne): kernel_constriant + // TODO(Rinne): recurrent_constriant + // TODO(Rinne): bias_constriant + Dropout = lstmArgs.Dropout, + RecurrentDropout = lstmArgs.RecurrentDropout, + Implementation = lstmArgs.Implementation, + DType = lstmArgs.DType, + Trainable = lstmArgs.Trainable + }); } - protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bool? training = null, IOptionalArgs? optional_args = null) { - return base.Call(inputs, initial_state: state, training: training); + // skip the condition of ragged input + + (inputs, initial_state, _) = _process_inputs(inputs, initial_state, null); + + Tensor mask = null; + if(optional_args is RnnOptionalArgs rnnArgs) + { + mask = rnnArgs.Mask; + } + + var single_input = inputs.Single; + var input_shape = single_input.shape; + var timesteps = args.TimeMajor ? input_shape[0] : input_shape[1]; + + _maybe_reset_cell_dropout_mask(Cell); + + Func step = (inputs, states) => + { + var res = Cell.Apply(inputs, states, training is null ? true : training.Value); + var (output, state) = res; + return (output, state); + }; + + var (last_output, outputs, states) = keras.backend.rnn( + step, + inputs, + initial_state, + constants: null, + go_backwards: args.GoBackwards, + mask: mask, + unroll: args.Unroll, + input_length: ops.convert_to_tensor(timesteps), + time_major: args.TimeMajor, + zero_output_for_mask: args.ZeroOutputForMask, + return_all_outputs: args.ReturnSequences + ); + + Tensor output; + if (args.ReturnSequences) + { + output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, args.GoBackwards); + } + else + { + output = last_output; + } + + if (args.ReturnState) + { + return new Tensor[] { output }.Concat(states).ToArray().ToTensors(); + } + else + { + return output; + } } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 17042767..bb71a914 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -1,5 +1,6 @@ using Serilog.Core; using System.Diagnostics; +using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; @@ -81,7 +82,7 @@ namespace Tensorflow.Keras.Layers.Rnn _bias_initializer = _args.BiasInitializer; } _bias = add_weight("bias", (_args.Units * 4), - initializer: _args.BiasInitializer); + initializer: _bias_initializer); } built = true; } @@ -94,7 +95,6 @@ namespace Tensorflow.Keras.Layers.Rnn var rec_dp_mask = get_recurrent_dropout_mask_for_cell( h_tm1, training.Value, count: 4); - Tensor c; Tensor o; if (_args.Implementation == 1) @@ -123,7 +123,7 @@ namespace Tensorflow.Keras.Layers.Rnn var x_f = math_ops.matmul(inputs_f, k_f); var x_c = math_ops.matmul(inputs_c, k_c); var x_o = math_ops.matmul(inputs_o, k_o); - if(_args.UseBias) + if (_args.UseBias) { var b = tf.split(_bias.AsTensor(), num_split: 4, axis: 0); Tensor b_i = b[0], b_f = b[1], b_c = b[2], b_o = b[3]; @@ -170,7 +170,7 @@ namespace Tensorflow.Keras.Layers.Rnn } var h = o * _args.Activation.Apply(c); // 这里是因为 Tensors 类初始化的时候会把第一个元素之后的元素打包成一个数组 - return new Tensors(h, h, c); + return new Nest(new INestStructure[] { new NestNode(h), new NestList(h, c) }).ToTensors(); } /// @@ -188,22 +188,21 @@ namespace Tensorflow.Keras.Layers.Rnn h_tm1_o = h_tm1[3]; var _recurrent_kernel_tensor = _recurrent_kernel.AsTensor(); - var startIndex = _recurrent_kernel_tensor.shape[0]; - var endIndex = _recurrent_kernel_tensor.shape[1]; + int startIndex = (int)_recurrent_kernel_tensor.shape[0]; var _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, new[] { 0, 0 }, new[] { startIndex, _args.Units }); var i = _args.RecurrentActivation.Apply( x_i + math_ops.matmul(h_tm1_i, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units }, new[] { startIndex, _args.Units * 2}); + new[] { 0, _args.Units }, new[] { startIndex, _args.Units}); var f = _args.RecurrentActivation.Apply( x_f + math_ops.matmul(h_tm1_f, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units * 2 }, new[] { startIndex, _args.Units * 3 }); + new[] { 0, _args.Units * 2 }, new[] { startIndex, _args.Units }); var c = f * c_tm1 + i * _args.Activation.Apply( x_c + math_ops.matmul(h_tm1_c, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, - new[] { 0, _args.Units * 3 }, new[] { startIndex, endIndex }); + new[] { 0, _args.Units * 3 }, new[] { startIndex, _args.Units }); var o = _args.RecurrentActivation.Apply( x_o + math_ops.matmul(h_tm1_o, _recurrent_kernel_slice)); diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0aeacc25..f86de8a8 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -45,23 +45,25 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public RNN(RNNArgs args) : base(PreConstruct(args)) + public RNN(IRnnCell cell, RNNArgs args) : base(PreConstruct(args)) { _args = args; SupportsMasking = true; - // if is StackedRnncell - if (args.Cells != null) - { - Cell = new StackedRNNCells(new StackedRNNCellsArgs - { - Cells = args.Cells - }); - } - else - { - Cell = args.Cell; - } + Cell = cell; + + // get input_shape + _args = PreConstruct(args); + + _num_constants = 0; + } + + public RNN(IEnumerable cells, RNNArgs args) : base(PreConstruct(args)) + { + _args = args; + SupportsMasking = true; + + Cell = new StackedRNNCells(cells, new StackedRNNCellsArgs()); // get input_shape _args = PreConstruct(args); @@ -330,7 +332,7 @@ namespace Tensorflow.Keras.Layers.Rnn states = new Tensors(states.SkipLast(_num_constants).ToArray()); states = len(states) == 1 && is_tf_rnn_cell ? new Tensors(states[0]) : states; var (output, new_states) = Cell.Apply(inputs, states, optional_args: new RnnOptionalArgs() { Constants = constants }); - return (output, new_states.Single); + return (output, new_states); }; } else @@ -382,6 +384,11 @@ namespace Tensorflow.Keras.Layers.Rnn } else { + //var tapeSet = tf.GetTapeSet(); + //foreach(var tape in tapeSet) + //{ + // tape.Watch(output); + //} return output; } } @@ -405,7 +412,7 @@ namespace Tensorflow.Keras.Layers.Rnn throw new NotImplementedException(); } - private (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensors inputs, Tensors initial_state, Tensors constants) + protected (Tensors inputs, Tensors initial_state, Tensors constants) _process_inputs(Tensors inputs, Tensors initial_state, Tensors constants) { if (inputs.Length > 1) { @@ -484,7 +491,7 @@ namespace Tensorflow.Keras.Layers.Rnn } - void _maybe_reset_cell_dropout_mask(ILayer cell) + protected void _maybe_reset_cell_dropout_mask(ILayer cell) { if (cell is DropoutRNNCellMixin CellDRCMixin) { @@ -495,26 +502,21 @@ namespace Tensorflow.Keras.Layers.Rnn private static RNNArgs PreConstruct(RNNArgs args) { - if (args.Kwargs == null) - { - args.Kwargs = new Dictionary(); - } - // If true, the output for masked timestep will be zeros, whereas in the // false case, output from previous timestep is returned for masked timestep. - var zeroOutputForMask = (bool)args.Kwargs.Get("zero_output_for_mask", false); + var zeroOutputForMask = args.ZeroOutputForMask; Shape input_shape; - var propIS = (Shape)args.Kwargs.Get("input_shape", null); - var propID = (int?)args.Kwargs.Get("input_dim", null); - var propIL = (int?)args.Kwargs.Get("input_length", null); + var propIS = args.InputShape; + var propID = args.InputDim; + var propIL = args.InputLength; if (propIS == null && (propID != null || propIL != null)) { input_shape = new Shape( propIL ?? -1, propID ?? -1); - args.Kwargs["input_shape"] = input_shape; + args.InputShape = input_shape; } return args; diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index 551c20cd..a22f31c7 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -10,14 +10,14 @@ namespace Tensorflow.Keras.Layers.Rnn public class SimpleRNN : RNN { SimpleRNNArgs args; - public SimpleRNN(SimpleRNNArgs args) : base(CreateCellForArgs(args)) + public SimpleRNN(SimpleRNNArgs args) : base(CreateCellForArgs(args), args) { this.args = args; } - private static SimpleRNNArgs CreateCellForArgs(SimpleRNNArgs args) + private static SimpleRNNCell CreateCellForArgs(SimpleRNNArgs args) { - args.Cell = new SimpleRNNCell(new SimpleRNNCellArgs() + return new SimpleRNNCell(new SimpleRNNCellArgs() { Units = args.Units, Activation = args.Activation, @@ -30,7 +30,6 @@ namespace Tensorflow.Keras.Layers.Rnn DType = args.DType, Trainable = args.Trainable, }); - return args; } } } \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 8fdc598e..c77f7779 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -115,10 +115,5 @@ namespace Tensorflow.Keras.Layers.Rnn return new Tensors(output, output); } } - - public Tensors get_initial_state(Tensors inputs = null, Tensor batch_size = null, TF_DataType dtype = TF_DataType.DtInvalid) - { - return RnnUtils.generate_zero_filled_state_for_cell(this, inputs, batch_size, dtype); - } } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 3e7b227c..8799bfb2 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -15,15 +15,11 @@ namespace Tensorflow.Keras.Layers.Rnn public IList Cells { get; set; } public bool _reverse_state_order; - public StackedRNNCells(StackedRNNCellsArgs args) : base(args) + public StackedRNNCells(IEnumerable cells, StackedRNNCellsArgs args) : base(args) { - if (args.Kwargs == null) - { - args.Kwargs = new Dictionary(); - } - Cells = args.Cells; - - _reverse_state_order = (bool)args.Kwargs.Get("reverse_state_order", false); + Cells = cells.ToList(); + + _reverse_state_order = args.ReverseStateOrder; if (_reverse_state_order) { diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 54ea1565..ed9b6ae9 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -55,30 +55,56 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual((2, 4), new_states[0].shape); } + [TestMethod] + public void TrainLSTMWithMnist() + { + var input = keras.Input((784)); + var x = keras.layers.Reshape((28, 28)).Apply(input); + //x = keras.layers.LSTM(50, return_sequences: true).Apply(x); + //x = keras.layers.LSTM(100, return_sequences: true).Apply(x); + //x = keras.layers.LSTM(150, return_sequences: true).Apply(x); + x = keras.layers.LSTM(4, implementation: 2).Apply(x); + //x = keras.layers.Dense(100).Apply(x); + var output = keras.layers.Dense(10, activation: "softmax").Apply(x); + + var model = keras.Model(input, output); + model.summary(); + model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + + var data_loader = new MnistModelLoader(); + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 58000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 30); + } + [TestMethod] public void SimpleRNN() { - //var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); - ///*var simple_rnn = keras.layers.SimpleRNN(4); - //var output = simple_rnn.Apply(inputs); - //Assert.AreEqual((32, 4), output.shape);*/ - - //var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); - //var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); - //Assert.AreEqual((6, 10, 4), whole_sequence_output.shape); - //Assert.AreEqual((6, 4), final_state.shape); + var input = keras.Input((784)); + var x = keras.layers.Reshape((28, 28)).Apply(input); + x = keras.layers.SimpleRNN(10).Apply(x); + var output = keras.layers.Dense(10, activation: "softmax").Apply(x); - var inputs = keras.Input(shape: (10, 8)); - var x = keras.layers.SimpleRNN(4).Apply(inputs); - var output = keras.layers.Dense(10).Apply(x); - var model = keras.Model(inputs, output); + var model = keras.Model(input, output); model.summary(); + model.compile(keras.optimizers.Adam(), keras.losses.CategoricalCrossentropy(), new string[] { "accuracy" }); - model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy()); - var datax = np.ones((16, 10, 8), dtype: dtypes.float32); - var datay = np.ones((16)); - model.fit(datax, datay, epochs: 20); + var data_loader = new MnistModelLoader(); + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 58000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 10); } + [TestMethod] public void RNNForSimpleRNNCell() { @@ -109,19 +135,5 @@ namespace Tensorflow.Keras.UnitTest.Layers Console.WriteLine($"output: {output}"); Assert.AreEqual((5, 4), output.shape); } - - [TestMethod] - public void MyTest() - { - var a = tf.zeros((2, 3)); - var b = tf.ones_like(a); - var c = tf.ones((3,4)); - - var d = new Tensors { a, b, c }; - var (A, BC) = d; - Console.WriteLine($"A:{A}"); - Console.WriteLine($"BC:{BC}"); - } - } } diff --git a/tools/Tensorflow.CodeGen/OpClassifier.cs b/tools/Tensorflow.CodeGen/OpClassifier.cs index eaad3fec..2d22c5d2 100644 --- a/tools/Tensorflow.CodeGen/OpClassifier.cs +++ b/tools/Tensorflow.CodeGen/OpClassifier.cs @@ -9,7 +9,7 @@ namespace Tensorflow.CodeGen { public class OpClassifier { - private static readonly string _filenamePattern = @"^gen_[a-z]*_ops.py$"; + private static readonly string _filenamePattern = @"^gen_[a-z_]*_ops.py$"; private static readonly string _pythonFunctionPattern = @"def\s+(\w+\d*\w*)\((?:\s*\w+\s*(?:=\s*[\S]*)*,\s*)*\s*name=None\):"; private Dictionary> _opSet = new(); public Dictionary> OpSet => _opSet; diff --git a/tools/Tensorflow.CodeGen/Utils.cs b/tools/Tensorflow.CodeGen/Utils.cs index 19de6c0e..6c69b7f9 100644 --- a/tools/Tensorflow.CodeGen/Utils.cs +++ b/tools/Tensorflow.CodeGen/Utils.cs @@ -178,10 +178,25 @@ namespace Tensorflow.CodeGen else if (attr.Type == "list(shape)") { res.Add((attr.Name, "Shape[]", "NOVALUE")); + if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.List) + { + List exps = new(); + foreach (var value in attr.DefaultValue.List.Shape) + { + exps.Add($"new Shape({string.Join(", ", value.Dim.Select(x => x.Size))})"); + } + string expression = "new Shape[]{" + $"{string.Join(", ", exps)}" + "}"; + dynamicDefaultValues[attr.Name] = expression; + res.Add((attr.Name, "string[]", $"null")); + } + else + { + res.Add((attr.Name, "string[]", "NOVALUE")); + } } else if (attr.Type == "list(string)") { - if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.S) + if (attr.DefaultValue is not null && attr.DefaultValue.ValueCase == AttrValue.ValueOneofCase.List) { List values = new(); foreach (var value in attr.DefaultValue.List.S) From a0df8109f83c343b3fb92e70871e95e495974262 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Sun, 18 Jun 2023 03:45:11 +0800 Subject: [PATCH 14/77] fix: training LSTM does not align with tensorflow. --- src/TensorFlowNET.Core/Binding.Util.cs | 2 +- .../Eager/EagerRunner.TFE_TapeGradient.cs | 2 +- .../Eager/EagerTensor.ToString.cs | 7 +++++- .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 2 +- .../Keras/Layers/ILayersApi.cs | 2 +- src/TensorFlowNET.Core/NumPy/NDArrayRender.cs | 18 +++++++-------- .../Initializers/NpyLoadInitializer.cs | 22 +++++++++++++++++++ .../Tensorflow.Binding.csproj | 2 +- src/TensorFlowNET.Core/Training/Trackable.cs | 3 +-- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 7 +++--- .../Layers/Rnn/LSTMCell.cs | 11 ++++++---- .../Layers/Rnn.Test.cs | 17 ++++++-------- tools/Tensorflow.CodeGen/FunctionGenerator.cs | 8 +++++-- .../Tensorflow.CodeGen.csproj | 2 +- 14 files changed, 68 insertions(+), 37 deletions(-) create mode 100644 src/TensorFlowNET.Core/Operations/Initializers/NpyLoadInitializer.cs diff --git a/src/TensorFlowNET.Core/Binding.Util.cs b/src/TensorFlowNET.Core/Binding.Util.cs index 8df39334..c5705930 100644 --- a/src/TensorFlowNET.Core/Binding.Util.cs +++ b/src/TensorFlowNET.Core/Binding.Util.cs @@ -503,7 +503,7 @@ namespace Tensorflow case Tensors tensors: return tensors.dtype; case IEnumerable tensors: - return tensors.First().dtype; + return tensors.Where(x => x is not null).First().dtype; case RefVariable variable: return variable.dtype; case ResourceVariable variable: diff --git a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs index 849dcb3f..3515fed8 100644 --- a/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs +++ b/src/TensorFlowNET.Core/Eager/EagerRunner.TFE_TapeGradient.cs @@ -65,7 +65,7 @@ namespace Tensorflow.Eager { outgrad_vec = output_gradients.ToList(); } - var result = tape.ComputeGradient(target_vec, sources_vec, source_tensors_that_are_targets, outgrad_vec, false); + var result = tape.ComputeGradient(target_vec, sources_vec, source_tensors_that_are_targets, outgrad_vec, true); bool unconnected_gradients_zero = unconnected_gradients == "zero"; diff --git a/src/TensorFlowNET.Core/Eager/EagerTensor.ToString.cs b/src/TensorFlowNET.Core/Eager/EagerTensor.ToString.cs index ce3c983b..71b3075a 100644 --- a/src/TensorFlowNET.Core/Eager/EagerTensor.ToString.cs +++ b/src/TensorFlowNET.Core/Eager/EagerTensor.ToString.cs @@ -10,6 +10,11 @@ namespace Tensorflow.Eager var str = NDArrayRender.ToString(nd); return $"tf.Tensor: shape={shape}, dtype={dtype.as_numpy_name()}, numpy={str}"; } - + public string ToString(int maxLength) + { + var nd = new NDArray(this); + var str = NDArrayRender.ToString(nd, maxLength); + return $"tf.Tensor: shape={shape}, dtype={dtype.as_numpy_name()}, numpy={str}"; + } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 1b26c05c..786236e4 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -29,7 +29,7 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn [JsonProperty("unit_forget_bias")] public bool UnitForgetBias { get; set; } = true; [JsonProperty("implementation")] - public int Implementation { get; set; } = 1; + public int Implementation { get; set; } = 2; } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 1eb08e77..a19508d4 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -182,7 +182,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 1, + int implementation = 2, bool return_sequences = false, bool return_state = false, bool go_backwards = false, diff --git a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs index 02cb5926..230797b8 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs @@ -7,7 +7,7 @@ namespace Tensorflow.NumPy { public class NDArrayRender { - public static string ToString(NDArray array) + public static string ToString(NDArray array, int maxLength = 10) { Shape shape = array.shape; if (shape.IsScalar) @@ -15,12 +15,12 @@ namespace Tensorflow.NumPy var s = new StringBuilder(); s.Append("array("); - Build(s, array); + Build(s, array, maxLength); s.Append(")"); return s.ToString(); } - static void Build(StringBuilder s, NDArray array) + static void Build(StringBuilder s, NDArray array, int maxLength) { var shape = array.shape; @@ -35,11 +35,11 @@ namespace Tensorflow.NumPy var len = shape[0]; s.Append("["); - if (len <= 10) + if (len <= maxLength) { for (int i = 0; i < len; i++) { - Build(s, array[i]); + Build(s, array[i], maxLength); if (i < len - 1) { s.Append(", "); @@ -49,9 +49,9 @@ namespace Tensorflow.NumPy } else { - for (int i = 0; i < 5; i++) + for (int i = 0; i < maxLength / 2; i++) { - Build(s, array[i]); + Build(s, array[i], maxLength); if (i < len - 1) { s.Append(", "); @@ -62,9 +62,9 @@ namespace Tensorflow.NumPy s.Append(" ... "); s.AppendLine(); - for (int i = (int)len - 5; i < len; i++) + for (int i = (int)len - maxLength / 2; i < len; i++) { - Build(s, array[i]); + Build(s, array[i], maxLength); if (i < len - 1) { s.Append(", "); diff --git a/src/TensorFlowNET.Core/Operations/Initializers/NpyLoadInitializer.cs b/src/TensorFlowNET.Core/Operations/Initializers/NpyLoadInitializer.cs new file mode 100644 index 00000000..202af652 --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/Initializers/NpyLoadInitializer.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.NumPy; + +namespace Tensorflow.Operations.Initializers +{ + /// + /// An initializer specially used for debugging (to load weights from disk). + /// + class NpyLoadInitializer : IInitializer + { + string _path; + public NpyLoadInitializer(string path) { _path = path; } + public string ClassName => ""; + public IDictionary Config => new Dictionary(); + public Tensor Apply(InitializerArgs args) + { + return np.load(_path); + } + } +} diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index b08b2e2b..02578ec1 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -111,7 +111,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Core/Training/Trackable.cs b/src/TensorFlowNET.Core/Training/Trackable.cs index 2b5bf2a7..3eff3487 100644 --- a/src/TensorFlowNET.Core/Training/Trackable.cs +++ b/src/TensorFlowNET.Core/Training/Trackable.cs @@ -179,8 +179,7 @@ namespace Tensorflow.Train // handles slot variables. if (!args.Overwrite || new_variable is RefVariable || new_variable is Trackable) { - var temp = new_variable as Trackable; - var res = _track_trackable(temp, args.Name, args.Overwrite); + var res = _track_trackable(new_variable as Trackable, args.Name, args.Overwrite); Debug.Assert(res is IVariableV1); return res as IVariableV1; } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index efca9300..0bdcbc84 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -793,7 +793,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 1) + int implementation = 2) => new LSTMCell(new LSTMCellArgs { Units = uints, @@ -846,7 +846,7 @@ namespace Tensorflow.Keras.Layers bool unit_forget_bias = true, float dropout = 0f, float recurrent_dropout = 0f, - int implementation = 1, + int implementation = 2, bool return_sequences = false, bool return_state = false, bool go_backwards = false, @@ -869,7 +869,8 @@ namespace Tensorflow.Keras.Layers GoBackwards = go_backwards, Stateful = stateful, TimeMajor = time_major, - Unroll = unroll + Unroll = unroll, + UnitForgetBias = unit_forget_bias }); /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index bb71a914..284a2b77 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -1,4 +1,5 @@ -using Serilog.Core; +using Newtonsoft.Json; +using Serilog.Core; using System.Diagnostics; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; @@ -54,6 +55,7 @@ namespace Tensorflow.Keras.Layers.Rnn public override void build(KerasShapesWrapper input_shape) { + base.build(input_shape); var single_shape = input_shape.ToSingleShape(); var input_dim = single_shape[-1]; _kernel = add_weight("kernel", (input_dim, _args.Units * 4), @@ -82,7 +84,8 @@ namespace Tensorflow.Keras.Layers.Rnn _bias_initializer = _args.BiasInitializer; } _bias = add_weight("bias", (_args.Units * 4), - initializer: _bias_initializer); + initializer: _bias_initializer + ); } built = true; } @@ -203,7 +206,7 @@ namespace Tensorflow.Keras.Layers.Rnn x_c + math_ops.matmul(h_tm1_c, _recurrent_kernel_slice)); _recurrent_kernel_slice = tf.slice(_recurrent_kernel_tensor, new[] { 0, _args.Units * 3 }, new[] { startIndex, _args.Units }); - var o = _args.RecurrentActivation.Apply( + var o = _args.Activation.Apply( x_o + math_ops.matmul(h_tm1_o, _recurrent_kernel_slice)); return new Tensors(c, o); @@ -220,7 +223,7 @@ namespace Tensorflow.Keras.Layers.Rnn Tensor z0 = z[0], z1 = z[1], z2 = z[2], z3 = z[3]; var i = _args.RecurrentActivation.Apply(z0); var f = _args.RecurrentActivation.Apply(z1); - var c = f * c_tm1 + i * _args.RecurrentActivation.Apply(z2); + var c = f * c_tm1 + i * _args.Activation.Apply(z2); var o = _args.RecurrentActivation.Apply(z3); return new Tensors(c, o); } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index ed9b6ae9..8eeee7a8 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -60,26 +60,23 @@ namespace Tensorflow.Keras.UnitTest.Layers { var input = keras.Input((784)); var x = keras.layers.Reshape((28, 28)).Apply(input); - //x = keras.layers.LSTM(50, return_sequences: true).Apply(x); - //x = keras.layers.LSTM(100, return_sequences: true).Apply(x); - //x = keras.layers.LSTM(150, return_sequences: true).Apply(x); - x = keras.layers.LSTM(4, implementation: 2).Apply(x); - //x = keras.layers.Dense(100).Apply(x); + x = keras.layers.LSTM(50, return_sequences: true).Apply(x); + x = keras.layers.LSTM(100).Apply(x); var output = keras.layers.Dense(10, activation: "softmax").Apply(x); var model = keras.Model(input, output); model.summary(); - model.compile(keras.optimizers.Adam(), keras.losses.SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + model.compile(keras.optimizers.Adam(), keras.losses.CategoricalCrossentropy(), new string[] { "accuracy" }); var data_loader = new MnistModelLoader(); var dataset = data_loader.LoadAsync(new ModelLoadSetting { TrainDir = "mnist", - OneHot = false, - ValidationSize = 58000, + OneHot = true, + ValidationSize = 55000, }).Result; - model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 30); + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 1); } [TestMethod] @@ -102,7 +99,7 @@ namespace Tensorflow.Keras.UnitTest.Layers ValidationSize = 58000, }).Result; - model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 10); + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size: 16, epochs: 2); } [TestMethod] diff --git a/tools/Tensorflow.CodeGen/FunctionGenerator.cs b/tools/Tensorflow.CodeGen/FunctionGenerator.cs index bb07dddf..f3687d6b 100644 --- a/tools/Tensorflow.CodeGen/FunctionGenerator.cs +++ b/tools/Tensorflow.CodeGen/FunctionGenerator.cs @@ -83,8 +83,12 @@ namespace Tensorflow.CodeGen sb.AppendLine("}"); // try - sb.Append("catch(NotOkStatusException ex)\n{\n"); - sb.AppendLine("throw ex;"); + sb.Append("catch(NotOkStatusException ex1)\n{\n"); + sb.AppendLine("throw ex1;"); + sb.AppendLine("}"); // catch + + sb.Append("catch(InvalidArgumentError ex2)\n{\n"); + sb.AppendLine("throw ex2;"); sb.AppendLine("}"); // catch sb.Append("catch(Exception)\n{\n"); diff --git a/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj b/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj index 4cb3368d..03195e6a 100644 --- a/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj +++ b/tools/Tensorflow.CodeGen/Tensorflow.CodeGen.csproj @@ -9,7 +9,7 @@ - + From 35d2e107f325dc0070cde780a9f8d491cfe2c4f8 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sun, 18 Jun 2023 12:15:56 +0800 Subject: [PATCH 15/77] refactor model.evaluate to deal with confilict --- src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 912f5e06..eaa9eb23 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -72,7 +72,7 @@ namespace Tensorflow.Keras.Engine { var data_handler = new DataHandler(new DataHandlerArgs { - X = new Tensors(x), + X = new Tensors(x.ToArray()), Y = y, Model = this, StepsPerExecution = _steps_per_execution @@ -168,7 +168,8 @@ namespace Tensorflow.Keras.Engine Dictionary test_step_multi_inputs_function(DataHandler data_handler, Tensor[] data) { var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = train_step(data_handler, new Tensors(data.Take(x_size)), new Tensors(data.Skip(x_size))); + var outputs = train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())); + tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } } From 1b1a50371b0829363d1f9c469aedbe727a6ec41f Mon Sep 17 00:00:00 2001 From: Visagan Guruparan <103048@smsassist.com> Date: Sun, 18 Jun 2023 22:46:36 -0500 Subject: [PATCH 16/77] np update square and dot product --- src/TensorFlowNET.Core/APIs/tf.math.cs | 15 ++++++++-- src/TensorFlowNET.Core/Binding.Util.cs | 23 ++++++++++++++- src/TensorFlowNET.Core/NumPy/Numpy.Math.cs | 21 ++++++++++++++ .../TensorFlowNET.UnitTest/Numpy/Math.Test.cs | 29 ++++++++++++++++++- 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 75253700..0e53d938 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -14,6 +14,7 @@ limitations under the License. ******************************************************************************/ +using Tensorflow.NumPy; using Tensorflow.Operations; namespace Tensorflow @@ -42,7 +43,6 @@ namespace Tensorflow public Tensor multiply(Tensor x, Tensor y, string name = null) => math_ops.multiply(x, y, name: name); - public Tensor divide_no_nan(Tensor a, Tensor b, string name = null) => math_ops.div_no_nan(a, b); @@ -452,7 +452,18 @@ namespace Tensorflow /// public Tensor multiply(Tx x, Ty y, string name = null) => gen_math_ops.mul(ops.convert_to_tensor(x), ops.convert_to_tensor(y), name: name); - + /// + /// return scalar product + /// + /// + /// + /// + /// + /// + /// + /// + public Tensor dot_prod(Tx x, Ty y, NDArray axes, string name = null) + => math_ops.tensordot(convert_to_tensor(x), convert_to_tensor(y), axes, name: name); public Tensor negative(Tensor x, string name = null) => gen_math_ops.neg(x, name); diff --git a/src/TensorFlowNET.Core/Binding.Util.cs b/src/TensorFlowNET.Core/Binding.Util.cs index 8df39334..e414ef6e 100644 --- a/src/TensorFlowNET.Core/Binding.Util.cs +++ b/src/TensorFlowNET.Core/Binding.Util.cs @@ -486,7 +486,28 @@ namespace Tensorflow throw new NotImplementedException(""); } } - + public static NDArray GetFlattenArray(NDArray x) + { + switch (x.GetDataType()) + { + case TF_DataType.TF_FLOAT: + x = x.ToArray(); + break; + case TF_DataType.TF_DOUBLE: + x = x.ToArray(); + break; + case TF_DataType.TF_INT16: + case TF_DataType.TF_INT32: + x = x.ToArray(); + break; + case TF_DataType.TF_INT64: + x = x.ToArray(); + break; + default: + break; + } + return x; + } public static TF_DataType GetDataType(this object data) { var type = data.GetType(); diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs index ea85048f..5bc97952 100644 --- a/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Math.cs @@ -49,9 +49,30 @@ namespace Tensorflow.NumPy [AutoNumPy] public static NDArray prod(params T[] array) where T : unmanaged => new NDArray(tf.reduce_prod(new NDArray(array))); + [AutoNumPy] + public static NDArray dot(NDArray x1, NDArray x2, NDArray? axes = null, string? name = null) + { + //if axes mentioned + if (axes != null) + { + return new NDArray(tf.dot_prod(x1, x2, axes, name)); + } + if (x1.shape.ndim > 1) + { + x1 = GetFlattenArray(x1); + } + if (x2.shape.ndim > 1) + { + x2 = GetFlattenArray(x2); + } + //if axes not mentioned, default 0,0 + return new NDArray(tf.dot_prod(x1, x2, axes: new int[] { 0, 0 }, name)); + } [AutoNumPy] public static NDArray power(NDArray x, NDArray y) => new NDArray(tf.pow(x, y)); + [AutoNumPy] + public static NDArray square(NDArray x) => new NDArray(tf.square(x)); [AutoNumPy] public static NDArray sin(NDArray x) => new NDArray(math_ops.sin(x)); diff --git a/test/TensorFlowNET.UnitTest/Numpy/Math.Test.cs b/test/TensorFlowNET.UnitTest/Numpy/Math.Test.cs index 32b517e4..65cdaedd 100644 --- a/test/TensorFlowNET.UnitTest/Numpy/Math.Test.cs +++ b/test/TensorFlowNET.UnitTest/Numpy/Math.Test.cs @@ -65,7 +65,34 @@ namespace TensorFlowNET.UnitTest.NumPy var y = np.power(x, 3); Assert.AreEqual(y, new[] { 0, 1, 8, 27, 64, 125 }); } - [TestMethod] + [TestMethod] + public void square() + { + var x = np.arange(6); + var y = np.square(x); + Assert.AreEqual(y, new[] { 0, 1, 4, 9, 16, 25 }); + } + [TestMethod] + public void dotproduct() + { + var x1 = new NDArray(new[] { 1, 2, 3 }); + var x2 = new NDArray(new[] { 4, 5, 6 }); + double result1 = np.dot(x1, x2); + NDArray y1 = new float[,] { + { 1.0f, 2.0f, 3.0f }, + { 4.0f, 5.1f,6.0f }, + { 4.0f, 5.1f,6.0f } + }; + NDArray y2 = new float[,] { + { 3.0f, 2.0f, 1.0f }, + { 6.0f, 5.1f, 4.0f }, + { 6.0f, 5.1f, 4.0f } + }; + double result2 = np.dot(y1, y2); + Assert.AreEqual(result1, 32); + Assert.AreEqual(Math.Round(result2, 2), 158.02); + } + [TestMethod] public void maximum() { var x1 = new NDArray(new[,] { { 1, 2, 3 }, { 4, 5.1, 6 } }); From 51b5f17c9a17397d61d1dc7df460517940e1107b Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Wed, 21 Jun 2023 21:41:06 +0800 Subject: [PATCH 17/77] fix: RNN training error on linux. --- src/TensorFlowNET.Core/APIs/c_api.cs | 14 ++++------- .../APIs/c_api.customize.cs | 2 +- src/TensorFlowNET.Core/Eager/GraphOnlyOps.cs | 25 +++++++++++++++++++ src/TensorFlowNET.Core/Graphs/FuncGraph.cs | 12 ++++----- src/TensorFlowNET.Core/Operations/list_ops.cs | 2 +- src/TensorFlowNET.Core/Operations/while_v2.cs | 9 +++---- src/TensorFlowNET.Core/ops.cs | 3 ++- 7 files changed, 44 insertions(+), 23 deletions(-) create mode 100644 src/TensorFlowNET.Core/Eager/GraphOnlyOps.cs diff --git a/src/TensorFlowNET.Core/APIs/c_api.cs b/src/TensorFlowNET.Core/APIs/c_api.cs index 6049c95c..d4744e78 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.cs @@ -51,17 +51,13 @@ namespace Tensorflow return handle == IntPtr.Zero ? String.Empty : Marshal.PtrToStringAnsi(handle); } - public unsafe static byte[] ByteStringPiece(IntPtr handle) + public unsafe static byte[] ByteStringPiece(Buffer? handle) { - byte* str_data = (byte*)handle.ToPointer(); - List bytes = new List(); - byte current = 255; - while (current != ((byte)'\0')) - { - current = *(str_data++); - bytes.Add(current); + if(handle is null){ + return new byte[0]; } - return bytes.Take(bytes.Count - 1).ToArray(); + var data = handle.ToArray(); + return data; } [UnmanagedFunctionPointer(CallingConvention.Winapi)] diff --git a/src/TensorFlowNET.Core/APIs/c_api.customize.cs b/src/TensorFlowNET.Core/APIs/c_api.customize.cs index d2aab9ac..510e52eb 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.customize.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.customize.cs @@ -10,7 +10,7 @@ namespace Tensorflow [DllImport(TensorFlowLibName)] public static extern void TFC_SetAttr(SafeGraphHandle graph, IntPtr op, string attr_name, SafeBufferHandle attr_value_proto, SafeStatusHandle status); [DllImport(TensorFlowLibName)] - public static extern IntPtr TFC_GetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output); + public static extern SafeBufferHandle TFC_GetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output); [DllImport(TensorFlowLibName)] public static extern void TFC_SetHandleShapeAndType(SafeGraphHandle c_graph, TF_Output output, byte[] data, long proto_len, SafeStatusHandle status); } diff --git a/src/TensorFlowNET.Core/Eager/GraphOnlyOps.cs b/src/TensorFlowNET.Core/Eager/GraphOnlyOps.cs new file mode 100644 index 00000000..2c20cfe9 --- /dev/null +++ b/src/TensorFlowNET.Core/Eager/GraphOnlyOps.cs @@ -0,0 +1,25 @@ +using Tensorflow; + +internal static class GraphOnlyOps +{ + /// + /// Graph-only version of tf.compat.v1.placeholder(), for internal use only. + /// + /// + /// + /// + /// + internal static Tensor graph_placeholder(TF_DataType dtype, Shape shape, string? name = null) + { + var dtype_value = new AttrValue() { Type = dtype.as_datatype_enum() }; + var shape_value = new AttrValue() { Shape = shape.as_proto() }; + var g = ops.get_default_graph(); + Dictionary attrs = new(); + attrs["dtype"] = dtype_value; + attrs["shape"] = shape_value; + var op = g.create_op("Placeholder", new Tensor[0], new TF_DataType[] { dtype }, + new TF_DataType[0], attrs: attrs, name: name); + var result = op.outputs[0]; + return result; + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs index ba7d7068..6f7fa9c5 100644 --- a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs +++ b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs @@ -544,12 +544,12 @@ public class FuncGraph : Graph, IDisposable Tensor placeholder; try { - placeholder = tf.placeholder(tensor.dtype, tensor.shape, name); + placeholder = GraphOnlyOps.graph_placeholder(tensor.dtype, tensor.shape, name); } - catch (ValueError) + catch (ValueError ex) { - // TODO(Rinne): Add warning here. - placeholder = tf.placeholder(tensor.dtype, tensor.shape); + tf.Logger.Warning(ex.ToString()); + placeholder = GraphOnlyOps.graph_placeholder(tensor.dtype, tensor.shape); } handle_data_util.copy_handle_data(tensor, placeholder); if (name is not null) @@ -575,12 +575,12 @@ public class FuncGraph : Graph, IDisposable Tensor placeholder; try { - placeholder = tf.placeholder(spec.dtype, spec.shape, requested_name); + placeholder = GraphOnlyOps.graph_placeholder(spec.dtype, spec.shape, requested_name); } catch (ValueError) { // TODO(Rinne): Add warning here. - placeholder = tf.placeholder(spec.dtype, spec.shape); + placeholder = GraphOnlyOps.graph_placeholder(spec.dtype, spec.shape); } if (name is not null) { diff --git a/src/TensorFlowNET.Core/Operations/list_ops.cs b/src/TensorFlowNET.Core/Operations/list_ops.cs index c5e83ee4..3791a2c1 100644 --- a/src/TensorFlowNET.Core/Operations/list_ops.cs +++ b/src/TensorFlowNET.Core/Operations/list_ops.cs @@ -31,7 +31,7 @@ namespace Tensorflow.Operations } else { - return ops.convert_to_tensor(shape); + return ops.convert_to_tensor(shape, dtype: dtypes.int32); } } diff --git a/src/TensorFlowNET.Core/Operations/while_v2.cs b/src/TensorFlowNET.Core/Operations/while_v2.cs index 3f324f87..aae15b77 100644 --- a/src/TensorFlowNET.Core/Operations/while_v2.cs +++ b/src/TensorFlowNET.Core/Operations/while_v2.cs @@ -38,9 +38,9 @@ namespace Tensorflow.Operations int len_orig_loop_vars = orig_loop_vars.Length; loop_vars = _tensor_array_to_flow(loop_vars); - loop_vars = Nest.MapStructure(x => _convert_to_tensor_or_indexed_slices(x, TF_DataType.DtInvalid, null), loop_vars).ToTensors(); + loop_vars = Nest.MapStructure(x => _convert_to_tensor_or_indexed_slices(x), loop_vars).ToTensors(); - var loop_vars_signature = Nest.MapStructure(x => new TensorSpec(x.shape, x.dtype), _tensor_array_to_flow(loop_vars)); + var loop_vars_signature = Nest.MapStructure(x => new TensorSpec(x.shape, x.dtype), loop_vars); var flat_shape_invariants = Nest.Flatten(loop_vars_signature).Select(x => x.shape).ToArray(); @@ -379,10 +379,9 @@ namespace Tensorflow.Operations return cond_graph.unique_name(cond_graph.Name + "___redundant_placeholder"); } - private static Tensor _convert_to_tensor_or_indexed_slices(Tensor value, TF_DataType dtype, - string name) + private static Tensor _convert_to_tensor_or_indexed_slices(Tensor value) { - return ops.convert_to_tensor(value, dtype, name, false); + return ops.convert_to_tensor(value, as_ref: false); } private static Tensor _build_maximum_iterations_loop_var(int maximum_iterations = -1) diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index fb9bccf3..a962e6d8 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -576,7 +576,8 @@ namespace Tensorflow public static HandleData get_resource_handle_data(Tensor graph_op) { var handle_data = c_api.TFC_GetHandleShapeAndType(graph_op.graph.c_graph, graph_op._as_tf_output()); - return HandleData.Parser.ParseFrom(c_api.ByteStringPiece(handle_data)); + var handle_str = c_api.ByteStringPiece(handle_data.DangerousGetHandle() == IntPtr.Zero ? null : new Buffer(handle_data)); + return HandleData.Parser.ParseFrom(handle_str); } public static void dismantle_graph(Graph graph) From 69b3bce3309d62b26d91614a1e2430ff0e5b183c Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Thu, 22 Jun 2023 02:07:10 +0800 Subject: [PATCH 18/77] test: update the redist version of test. --- .../Tensorflow.UnitTest.RedistHolder.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj b/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj index 87807758..1ca387db 100644 --- a/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj +++ b/tools/Tensorflow.UnitTest.RedistHolder/Tensorflow.UnitTest.RedistHolder.csproj @@ -5,7 +5,7 @@ - + From 46e216279747397507f833e765843467c6f35e40 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 21 Jun 2023 17:25:17 -0500 Subject: [PATCH 19/77] Fix model.evaluate in NeuralNetXorKeras. --- src/TensorFlowNET.Core/APIs/c_api.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/TensorFlowNET.Core/APIs/c_api.cs b/src/TensorFlowNET.Core/APIs/c_api.cs index 6049c95c..63bdfd27 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.cs @@ -53,6 +53,11 @@ namespace Tensorflow public unsafe static byte[] ByteStringPiece(IntPtr handle) { + if (handle == IntPtr.Zero) + { + return new byte[0]; + } + byte* str_data = (byte*)handle.ToPointer(); List bytes = new List(); byte current = 255; From ae8fe840e457b0b34d04fc0cafdb31d89b7a9d4d Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Thu, 22 Jun 2023 09:21:18 +0800 Subject: [PATCH 20/77] fix: resolve conflict. --- src/TensorFlowNET.Core/APIs/c_api.cs | 4 +++- src/TensorFlowNET.Core/ops.cs | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/c_api.cs b/src/TensorFlowNET.Core/APIs/c_api.cs index 559176a5..a91b8682 100644 --- a/src/TensorFlowNET.Core/APIs/c_api.cs +++ b/src/TensorFlowNET.Core/APIs/c_api.cs @@ -53,8 +53,10 @@ namespace Tensorflow public unsafe static byte[] ByteStringPiece(Buffer? handle) { - if(handle is null){ + if (handle is null) + { return new byte[0]; + } var data = handle.ToArray(); return data; } diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index a962e6d8..7bd78a79 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -576,8 +576,14 @@ namespace Tensorflow public static HandleData get_resource_handle_data(Tensor graph_op) { var handle_data = c_api.TFC_GetHandleShapeAndType(graph_op.graph.c_graph, graph_op._as_tf_output()); - var handle_str = c_api.ByteStringPiece(handle_data.DangerousGetHandle() == IntPtr.Zero ? null : new Buffer(handle_data)); - return HandleData.Parser.ParseFrom(handle_str); + try{ + var handle_str = c_api.ByteStringPiece(handle_data.DangerousGetHandle() == IntPtr.Zero ? null : new Buffer(handle_data)); + return HandleData.Parser.ParseFrom(handle_str); + } + catch(Exception){ + var handle_str = c_api.ByteStringPieceFromNativeString(handle_data.DangerousGetHandle()); + return HandleData.Parser.ParseFrom(handle_str); + } } public static void dismantle_graph(Graph graph) From 4c6063d03e3bb8af35007c16ca2585c772994301 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 21 Jun 2023 21:05:51 -0500 Subject: [PATCH 21/77] Update version number. --- src/TensorFlowNET.Core/Tensorflow.Binding.csproj | 10 +++++----- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 8 ++++---- test/TensorflowNET.Hub.Unittest/KerasLayerTest.cs | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 02578ec1..61b86168 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.10.0 - 0.100.5 + 0.110.0 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.100.5.0 + 0.110.0.0 tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -38,7 +38,7 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. - 0.100.5.0 + 0.110.0.0 LICENSE true packages @@ -110,13 +110,13 @@ https://tensorflownet.readthedocs.io - + - + diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 8b3c9265..320c3b67 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.10.5 + 0.11.0 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2023 @@ -38,8 +38,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.10.5.0 - 0.10.5.0 + 0.11.0.0 + 0.11.0.0 LICENSE Debug;Release;GPU @@ -71,7 +71,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + diff --git a/test/TensorflowNET.Hub.Unittest/KerasLayerTest.cs b/test/TensorflowNET.Hub.Unittest/KerasLayerTest.cs index 4ee4d54c..b9a8ed80 100644 --- a/test/TensorflowNET.Hub.Unittest/KerasLayerTest.cs +++ b/test/TensorflowNET.Hub.Unittest/KerasLayerTest.cs @@ -6,6 +6,7 @@ namespace Tensorflow.Hub.Unittest [TestClass] public class KerasLayerTest { + [Ignore] [TestMethod] public void SmallBert() { From 3805771121162c3e0806198acd18619c6cd6394b Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Thu, 22 Jun 2023 05:53:10 +0000 Subject: [PATCH 22/77] improve layer norm --- src/TensorFlowNET.Core/APIs/tf.nn.cs | 18 +++++++++++++++ .../Normalization/LayerNormalization.cs | 15 ++++++++++++- .../Layers/LayersTest.cs | 22 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.nn.cs b/src/TensorFlowNET.Core/APIs/tf.nn.cs index e0c29bfa..08b88c3d 100644 --- a/src/TensorFlowNET.Core/APIs/tf.nn.cs +++ b/src/TensorFlowNET.Core/APIs/tf.nn.cs @@ -14,8 +14,10 @@ limitations under the License. ******************************************************************************/ +using System.Xml.Linq; using Tensorflow.Operations; using Tensorflow.Operations.Activation; +//using static System.Formats.Asn1.AsnWriter; using static Tensorflow.Binding; namespace Tensorflow @@ -125,6 +127,22 @@ namespace Tensorflow is_training: is_training, name: name, exponential_avg_factor: exponential_avg_factor); + public Tensor batch_normalization(Tensor x, + Tensor mean, + Tensor variance, + Tensor offset, + Tensor scale, + float variance_epsilon, + string name = null) + { + var inv = math_ops.rsqrt(variance + variance_epsilon); + tf_with(ops.name_scope(name, "batchnorm", (x, mean, variance, scale, offset)), scope => + { + if (scale != null) inv *= scale; + }); + if (offset != null) return x * math_ops.cast(inv, x.dtype) + math_ops.cast(offset - mean * inv, dtype: x.dtype); + else return x * math_ops.cast(inv, x.dtype) + math_ops.cast(-mean * inv, dtype: x.dtype); + } public Tensor max_pool(Tensor value, int[] ksize, int[] strides, string padding, string data_format = "NHWC", string name = null) => nn_ops.max_pool(value, ksize, strides, padding, data_format: data_format, name: name); diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs index 1898f24c..69bdfbaa 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs @@ -153,9 +153,22 @@ namespace Tensorflow.Keras.Layers } else { + var input_dtype = inputs.dtype; + if ((input_dtype == tf.float16) && DType == tf.float32) inputs = tf.cast(inputs, tf.float32); + (Tensor mean, Tensor variance) = tf.nn.moments(inputs, axis, keep_dims: true); - } + (Tensor scale, Tensor offset) = (_broadcast(gamma), _broadcast(beta)); + + outputs = tf.nn.batch_normalization( + inputs, + mean, + variance, + offset: offset, + scale: scale, + variance_epsilon: epsilon); + outputs = tf.cast(outputs, input_dtype); + } // If some components of the shape got lost due to adjustments, fix that. outputs.shape = input_shape; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index f4980b82..98d90966 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -1,5 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using System.Collections.Generic; +using System.Linq; using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -161,6 +163,26 @@ namespace Tensorflow.Keras.UnitTest.Layers Tensor output = layer.Apply(inputs); Assert.AreEqual((5, 2), output.shape); Assert.IsTrue(output[0].numpy().Equals(new[] { -0.99998f, 0.99998f })); + + // test_layernorm_weights + Assert.AreEqual(len(layer.TrainableWeights), 2); + Assert.AreEqual(len(layer.Weights), 2); + + var beta = layer.Weights.Where(x => x.Name.StartsWith("beta")).Single(); + var gamma = layer.Weights.Where(x => x.Name.StartsWith("gamma")).Single(); + + // correctness_test + layer = keras.layers.LayerNormalization(axis: -1, epsilon: (float) 1e-12); + var x = np.random.normal(loc: 5.0f, scale: 10.0f, size: (1000, 2, 2, 2)).astype(tf.float32); + + output = layer.Apply(x); + + var y = (output - beta.numpy()) / gamma.numpy(); + + var y_mean = np.mean(y.numpy()); + var y_std = np.sqrt(np.sum(np.power(y.numpy() - np.mean(y.numpy()), 2)) / 8000); + Assert.IsTrue(tf.greater(np.array(0.1f), tf.abs(y_std - 1.0)).ToArray()[0]); + Assert.IsTrue(tf.greater(np.array(0.1f), tf.abs(y_mean)).ToArray()[0]); } /// From 786b26602ff502284f56d85586961fb9f824cc22 Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Thu, 22 Jun 2023 07:15:08 +0000 Subject: [PATCH 23/77] Modify according to the reviewer's comments --- src/TensorFlowNET.Core/APIs/tf.nn.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.nn.cs b/src/TensorFlowNET.Core/APIs/tf.nn.cs index 08b88c3d..e5cd4e56 100644 --- a/src/TensorFlowNET.Core/APIs/tf.nn.cs +++ b/src/TensorFlowNET.Core/APIs/tf.nn.cs @@ -17,7 +17,6 @@ using System.Xml.Linq; using Tensorflow.Operations; using Tensorflow.Operations.Activation; -//using static System.Formats.Asn1.AsnWriter; using static Tensorflow.Binding; namespace Tensorflow @@ -127,6 +126,18 @@ namespace Tensorflow is_training: is_training, name: name, exponential_avg_factor: exponential_avg_factor); + + /// + /// Normalizes a tensor by `mean` and `variance`, and applies (optionally) a`scale` \\(\gamma\\) to it, as well as an `offset` \\(\beta\\). + /// + /// A floating point tensor. + /// A mean `Tensor`. + /// A variance `Tensor`. + /// An offset `Tensor`, often denoted \\(\beta\\) in equations, or NULL. If present, will be added to the normalized tensor. + /// A scale `Tensor`, often denoted \\(\gamma\\) in equations, or NULL. If present, the scale is applied to the normalized tensor. + /// A small float number to avoid dividing by 0. + /// A name for this operation. + /// the normalized, scaled, offset tensor. public Tensor batch_normalization(Tensor x, Tensor mean, Tensor variance, From fcd10447abb20e50ed2d67e313c2f75566319649 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 23 Jun 2023 13:39:36 +0800 Subject: [PATCH 24/77] add more type case for tensor.zeros --- src/TensorFlowNET.Core/Operations/array_ops.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index a0b47aac..24c39215 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -84,8 +84,13 @@ namespace Tensorflow // var shape_tensor = constant_op._tensor_shape_tensor_conversion_function(shape); Tensor zeros = dtype switch { + TF_DataType.TF_BOOL => constant(false), TF_DataType.TF_DOUBLE => constant(0d), TF_DataType.TF_FLOAT => constant(0f), + TF_DataType.TF_INT64 => constant(0L), + TF_DataType.TF_UINT64 => constant((ulong)0), + TF_DataType.TF_INT32 => constant(0), + TF_DataType.TF_UINT32 => constant((uint)0), TF_DataType.TF_INT8 => constant((sbyte)0), TF_DataType.TF_UINT8 => constant((byte)0), _ => constant(0) @@ -108,9 +113,15 @@ namespace Tensorflow return _constant_if_small(0.0F, shape, dtype, name); case TF_DataType.TF_INT64: return _constant_if_small(0L, shape, dtype, name); + case TF_DataType.TF_UINT64: + return _constant_if_small(0, shape, dtype, name); case TF_DataType.TF_INT32: return _constant_if_small(0, shape, dtype, name); + case TF_DataType.TF_UINT32: + return _constant_if_small(0, shape, dtype, name); case TF_DataType.TF_INT8: + return _constant_if_small(0, shape, dtype, name); + case TF_DataType.TF_UINT8: return _constant_if_small(0, shape, dtype, name); default: throw new TypeError("can't find type for zeros"); From e749aaeaae197464f817e1c7bfffe6f922d55b6a Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 23 Jun 2023 14:04:44 +0800 Subject: [PATCH 25/77] add more implicit operator for NDArray and UnitTest for `keras.datasets.imdb` --- src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs | 6 ++++++ .../TensorFlowNET.UnitTest/Dataset/DatasetTest.cs | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs b/src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs index fd4f93fc..45b236c7 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs @@ -107,9 +107,15 @@ namespace Tensorflow.NumPy public static implicit operator NDArray(bool value) => new NDArray(value); + public static implicit operator NDArray(byte value) + => new NDArray(value); + public static implicit operator NDArray(int value) => new NDArray(value); + public static implicit operator NDArray(long value) + => new NDArray(value); + public static implicit operator NDArray(float value) => new NDArray(value); diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index 8317346e..875e5001 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -2,6 +2,7 @@ using System; using System.Linq; using static Tensorflow.Binding; +using static Tensorflow.KerasApi; namespace TensorFlowNET.UnitTest.Dataset { @@ -195,5 +196,19 @@ namespace TensorFlowNET.UnitTest.Dataset Assert.IsFalse(allEqual); } + [TestMethod] + public void GetData() + { + var vocab_size = 20000; + var dataset = keras.datasets.imdb.load_data(num_words: vocab_size); + var x_train = dataset.Train.Item1; + Assert.AreEqual(x_train.dims[0], 25000); + var y_train = dataset.Train.Item2; + Assert.AreEqual(y_train.dims[0], 25000); + var x_val = dataset.Test.Item1; + Assert.AreEqual(x_val.dims[0], 25000); + var y_val = dataset.Test.Item2; + Assert.AreEqual(y_val.dims[0], 25000); + } } } From c23b24633fa1111d613deeedba5c9869ea463dd8 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 23 Jun 2023 14:21:27 +0800 Subject: [PATCH 26/77] remove UnitTest for `keras.datasets.imdb` --- .../TensorFlowNET.UnitTest/Dataset/DatasetTest.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs index 875e5001..8317346e 100644 --- a/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs +++ b/test/TensorFlowNET.UnitTest/Dataset/DatasetTest.cs @@ -2,7 +2,6 @@ using System; using System.Linq; using static Tensorflow.Binding; -using static Tensorflow.KerasApi; namespace TensorFlowNET.UnitTest.Dataset { @@ -196,19 +195,5 @@ namespace TensorFlowNET.UnitTest.Dataset Assert.IsFalse(allEqual); } - [TestMethod] - public void GetData() - { - var vocab_size = 20000; - var dataset = keras.datasets.imdb.load_data(num_words: vocab_size); - var x_train = dataset.Train.Item1; - Assert.AreEqual(x_train.dims[0], 25000); - var y_train = dataset.Train.Item2; - Assert.AreEqual(y_train.dims[0], 25000); - var x_val = dataset.Test.Item1; - Assert.AreEqual(x_val.dims[0], 25000); - var y_val = dataset.Test.Item2; - Assert.AreEqual(y_val.dims[0], 25000); - } } } From bfa9f77f42a361b4a31b644454d6338182c81e93 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 24 Jun 2023 08:55:40 -0500 Subject: [PATCH 27/77] tf.math.sqrt --- src/TensorFlowNET.Core/APIs/tf.math.cs | 2 +- src/TensorFlowNET.Core/Operations/math_ops.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 0e53d938..ffbc4373 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -354,7 +354,7 @@ namespace Tensorflow => a / b; public Tensor sqrt(Tensor a, string name = null) - => gen_math_ops.sqrt(a, name); + => math_ops.sqrt(a, name); public Tensor sign(Tensor a, string name = null) => gen_math_ops.sign(a, name); diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 5ded448a..d00a5d36 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -269,7 +269,7 @@ namespace Tensorflow => tf.Context.ExecuteOp("Erf", name, new ExecuteOpArgs(x)); public static Tensor sqrt(Tensor x, string name = null) - => gen_math_ops.sqrt(x, name: name); + => tf.Context.ExecuteOp("Sqrt", name, new ExecuteOpArgs(x)); public static Tensor multiply(Tensor x, Tensor y, string name = null) => tf.Context.ExecuteOp("Mul", name, new ExecuteOpArgs(x, y)); From eeb20e4fe620161a2e65ce63e72cd39cd9086548 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Mon, 26 Jun 2023 16:20:45 +0800 Subject: [PATCH 28/77] Add new feature: Add UpSampling1D layer and test. --- .../Reshaping/UpSampling2DArgs.cs | 2 +- .../Reshaping/Upsampling1DArgs.cs | 10 +++ .../Keras/Layers/ILayersApi.Reshaping.cs | 4 ++ src/TensorFlowNET.Keras/BackendImpl.cs | 26 ++++++++ .../Layers/LayersApi.Reshaping.cs | 61 +++++++++++-------- .../Layers/Reshaping/UpSampling1D.cs | 32 ++++++++++ .../Layers/Reshaping/UpSampling2D.cs | 3 + .../Layers/Layers.Reshaping.Test.cs | 10 +++ 8 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Upsampling1DArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling1D.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs index b35e0e4b..504b3d46 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs @@ -7,7 +7,7 @@ namespace Tensorflow.Keras.ArgsDefinition [JsonProperty("size")] public Shape Size { get; set; } [JsonProperty("data_format")] - public string DataFormat { get; set; } + public string DataFormat { get; set; } = "channels_last"; /// /// 'nearest', 'bilinear' /// diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Upsampling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Upsampling1DArgs.cs new file mode 100644 index 00000000..4e3dbf17 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Upsampling1DArgs.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class UpSampling1DArgs : AutoSerializeLayerArgs + { + [JsonProperty("size")] + public int Size { get; set; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs index d41e0688..ae34c514 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs @@ -9,6 +9,10 @@ namespace Tensorflow.Keras.Layers public ILayer Reshape(Shape target_shape); public ILayer Reshape(object[] target_shape); + public ILayer UpSampling1D( + int size + ); + public ILayer UpSampling2D(Shape size = null, string data_format = null, string interpolation = "nearest"); diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 8dbcf90d..364800ae 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -956,6 +956,32 @@ namespace Tensorflow.Keras } + /// + /// Repeats the elements of a tensor along an axis, like `np.repeat`. + /// + /// + /// + /// + /// + public Tensor repeat_elements(Tensor x, int rep, int axis) + { + var x_shape = x.shape.as_int_list(); + if (x_shape[axis] != -1) + { + var splits = tf.split(x, x_shape[axis], axis:axis); + var x_rep = splits.SelectMany(s => Enumerable.Repeat(s, rep)).ToArray(); + return concatenate(x_rep, axis); + } + //var auxiliary_axis = axis + 1; + //x_shape = x.shape; + //var x_rep = tf.expand_dims(x, auxiliary_axis); + //var reps = np.ones(x_shape.Length + 1); + //reps[auxiliary_axis] = rep; + //x_rep = tf.tile(x_rep, reps); + + throw new NotImplementedException(); + + } public Tensor reverse(Tensor input, int axis) { return reverse(input, new int[] { axis }); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs index d3db1d66..2ee99bc7 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs @@ -6,35 +6,48 @@ using Tensorflow.Keras.ArgsDefinition; namespace Tensorflow.Keras.Layers { public partial class LayersApi { - /// - /// Zero-padding layer for 2D input (e.g. picture). - /// - /// - /// - public ILayer ZeroPadding2D ( NDArray padding ) + + /// + /// Upsampling layer for 1D inputs. Repeats each temporal step `size` times along the time axis. + /// + /// + /// + public ILayer UpSampling1D(int size) + => new UpSampling1D(new UpSampling1DArgs + { + Size = size + }); + + /// + /// Zero-padding layer for 2D input (e.g. picture). + /// + /// + /// + public ILayer ZeroPadding2D ( NDArray padding ) => new ZeroPadding2D(new ZeroPadding2DArgs { Padding = padding }); - /// - /// Upsampling layer for 2D inputs.
- /// Repeats the rows and columns of the data by size[0] and size[1] respectively. - /// - /// - /// - /// - /// - public ILayer UpSampling2D ( Shape size = null, - string data_format = null, - string interpolation = "nearest" ) - => new UpSampling2D(new UpSampling2DArgs { - Size = size ?? (2, 2) - }); + /// + /// Upsampling layer for 2D inputs.
+ /// Repeats the rows and columns of the data by size[0] and size[1] respectively. + /// + /// + /// + /// + /// + public ILayer UpSampling2D(Shape size, string data_format, string interpolation) + => new UpSampling2D(new UpSampling2DArgs + { + Size = size, + DataFormat = data_format, + Interpolation = interpolation + }); - /// - /// Permutes the dimensions of the input according to a given pattern. - /// - public ILayer Permute ( int[] dims ) + /// + /// Permutes the dimensions of the input according to a given pattern. + /// + public ILayer Permute ( int[] dims ) => new Permute(new PermuteArgs { dims = dims }); diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling1D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling1D.cs new file mode 100644 index 00000000..3bc8d6c6 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling1D.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; + + +namespace Tensorflow.Keras.Layers +{ + /// + /// Upsampling layer for 1D inputs. + /// + public class UpSampling1D : Layer + { + UpSampling1DArgs args; + int size; + + public UpSampling1D(UpSampling1DArgs args) : base(args) + { + this.args = args; + size = args.Size; + inputSpec = new InputSpec(ndim: 3); + } + + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + var output = keras.backend.repeat_elements(inputs, size, axis: 1); + return output; + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs index 223f33d4..cb579d61 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/UpSampling2D.cs @@ -10,6 +10,9 @@ using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers { + /// + /// Upsampling layer for 2D inputs. + /// public class UpSampling2D : Layer { UpSampling2DArgs args; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Reshaping.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Reshaping.Test.cs index 748544cb..5b16cc90 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Reshaping.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Reshaping.Test.cs @@ -1,4 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -18,6 +19,15 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual((1, 2, 3, 2), y.shape); } + [TestMethod] + public void UpSampling1D() + { + Shape input_shape = (2, 2, 3); + var x = np.arange(input_shape.size).reshape(input_shape); + var y = tf.keras.layers.UpSampling1D(size: 2).Apply(x); + Assert.AreEqual((2, 4, 3), y.shape); + } + [TestMethod] public void UpSampling2D() { From 61c927ac170fe0294191d4f9af876a4d00f41052 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 26 Jun 2023 09:24:45 -0500 Subject: [PATCH 29/77] Release v0.110.0. --- src/TensorFlowNET.Core/Tensorflow.Binding.csproj | 6 +++++- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 2 +- .../TensorFlowNET.Graph.UnitTest.csproj | 6 +++--- .../Tensorflow.Keras.UnitTest.csproj | 6 +++--- .../Tensorflow.Native.UnitTest.csproj | 6 +++--- .../Tensorflow.Binding.UnitTest.csproj | 6 +++--- tools/TensorFlowNET.Benchmarks/Tensorflow.Benchmark.csproj | 2 +- tools/TensorFlowNET.Console/Tensorflow.Console.csproj | 2 +- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 61b86168..3bc20289 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -10,7 +10,7 @@ enable Haiping Chen, Meinrad Recheis, Eli Belash SciSharp STACK - true + False Apache 2.0, Haiping Chen $([System.DateTime]::UtcNow.ToString(yyyy)) https://github.com/SciSharp/TensorFlow.NET git @@ -22,6 +22,9 @@ Building, training and infering deep learning models. https://tensorflownet.readthedocs.io 0.110.0.0 + tf.net 0.110.x and above are based on tensorflow native 2.11.0 + * RNN, LSTM works. + tf.net 0.100.x and above are based on tensorflow native 2.10.0 * Eager Mode is added finally. @@ -37,6 +40,7 @@ https://tensorflownet.readthedocs.io tf.net 0.6x.x aligns with TensorFlow v2.6.x native library. tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. + tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. 0.110.0.0 LICENSE diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 320c3b67..5dc46fe4 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -31,7 +31,7 @@ Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear & actionable error messages. SciSharp STACK - true + False tensorflow, keras, deep learning, machine learning true packages diff --git a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj index c353832a..78a0938c 100644 --- a/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj +++ b/test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj @@ -24,9 +24,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index d744c336..58c176e8 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -13,9 +13,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj b/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj index 9fec0e6d..a4f1ec56 100644 --- a/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj +++ b/test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj @@ -44,9 +44,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index 98dadf01..240960c9 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -42,9 +42,9 @@ - - - + + + diff --git a/tools/TensorFlowNET.Benchmarks/Tensorflow.Benchmark.csproj b/tools/TensorFlowNET.Benchmarks/Tensorflow.Benchmark.csproj index f2495d22..dd6f9538 100644 --- a/tools/TensorFlowNET.Benchmarks/Tensorflow.Benchmark.csproj +++ b/tools/TensorFlowNET.Benchmarks/Tensorflow.Benchmark.csproj @@ -37,7 +37,7 @@ - + diff --git a/tools/TensorFlowNET.Console/Tensorflow.Console.csproj b/tools/TensorFlowNET.Console/Tensorflow.Console.csproj index c79d4845..ecc2d30b 100644 --- a/tools/TensorFlowNET.Console/Tensorflow.Console.csproj +++ b/tools/TensorFlowNET.Console/Tensorflow.Console.csproj @@ -20,7 +20,7 @@ - + From fff5029b0240c30ee4f2b9329c71c8665e091858 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Tue, 27 Jun 2023 23:57:48 +0800 Subject: [PATCH 30/77] fix: revise earlystopping callback's min_delta parameter --- src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs index 73ccc87b..59152d9b 100644 --- a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs +++ b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs @@ -11,7 +11,7 @@ namespace Tensorflow.Keras.Callbacks; public class EarlyStopping: ICallback { int _paitence; - int _min_delta; + float _min_delta; int _verbose; int _stopped_epoch; int _wait; @@ -26,7 +26,7 @@ public class EarlyStopping: ICallback CallbackParams _parameters; public Dictionary>? history { get; set; } // user need to pass a CallbackParams to EarlyStopping, CallbackParams at least need the model - public EarlyStopping(CallbackParams parameters,string monitor = "val_loss", int min_delta = 0, int patience = 0, + public EarlyStopping(CallbackParams parameters,string monitor = "val_loss", float min_delta = 0f, int patience = 0, int verbose = 1, string mode = "auto", float baseline = 0f, bool restore_best_weights = false, int start_from_epoch = 0) { From 81b10e37809d5ae57989f55d4102a0a367d4322c Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Wed, 28 Jun 2023 02:19:28 +0800 Subject: [PATCH 31/77] feat: Add GRUCell layer --- src/TensorFlowNET.Core/APIs/tf.tensor.cs | 13 +- .../Keras/ArgsDefinition/Rnn/GRUCellArgs.cs | 39 +++ .../Keras/Layers/ILayersApi.cs | 12 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 43 +++ src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs | 282 ++++++++++++++++++ .../Layers/Rnn.Test.cs | 13 + 6 files changed, 399 insertions(+), 3 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.tensor.cs b/src/TensorFlowNET.Core/APIs/tf.tensor.cs index 45aebc0c..b03168ab 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tensor.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tensor.cs @@ -68,20 +68,27 @@ namespace Tensorflow /// A name for the operation (optional) /// if num_or_size_splits is a scalar returns num_or_size_splits Tensor objects; /// if num_or_size_splits is a 1-D Tensor returns num_or_size_splits.get_shape[0] Tensor objects resulting from splitting value. - public Tensor[] split(Tensor value, int num_split, Tensor axis, string name = null) + public Tensor[] split(Tensor value, int num_split, Axis axis, string name = null) => array_ops.split( value: value, num_or_size_splits: num_split, axis: axis, name: name); - public Tensor[] split(Tensor value, int num_split, int axis, string name = null) + public Tensor[] split(Tensor value, int[] num_split, Axis axis, string name = null) => array_ops.split( value: value, num_or_size_splits: num_split, - axis: ops.convert_to_tensor(axis), + axis: axis, name: name); + //public Tensor[] split(Tensor value, int num_split, Axis axis, string name = null) + // => array_ops.split( + // value: value, + // num_or_size_splits: num_split, + // axis: axis, + // name: name); + public Tensor ensure_shape(Tensor x, Shape shape, string name = null) { return gen_ops.ensure_shape(x, shape, name); diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs new file mode 100644 index 00000000..75d5d021 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs @@ -0,0 +1,39 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class GRUCellArgs : AutoSerializeLayerArgs + { + [JsonProperty("units")] + public int Units { get; set; } + // TODO(Rinne): lack of initialized value of Activation. Merging keras + // into tf.net could resolve it. + [JsonProperty("activation")] + public Activation Activation { get; set; } + [JsonProperty("recurrent_activation")] + public Activation RecurrentActivation { get; set; } + [JsonProperty("use_bias")] + public bool UseBias { get; set; } = true; + [JsonProperty("dropout")] + public float Dropout { get; set; } = .0f; + [JsonProperty("recurrent_dropout")] + public float RecurrentDropout { get; set; } = .0f; + [JsonProperty("kernel_initializer")] + public IInitializer KernelInitializer { get; set; } + [JsonProperty("recurrent_initializer")] + public IInitializer RecurrentInitializer { get; set; } + [JsonProperty("bias_initializer")] + public IInitializer BiasInitializer { get; set; } + [JsonProperty("reset_after")] + public bool ResetAfter { get;set; } + [JsonProperty("implementation")] + public int Implementation { get; set; } = 2; + + + + } + +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index a19508d4..9bc99701 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -246,6 +246,18 @@ namespace Tensorflow.Keras.Layers bool time_major = false ); + public IRnnCell GRUCell( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool reset_after = true); + public ILayer Subtract(); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 0bdcbc84..d2080337 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -873,6 +873,45 @@ namespace Tensorflow.Keras.Layers UnitForgetBias = unit_forget_bias }); + /// + /// Cell class for the GRU layer. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public IRnnCell GRUCell( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool reset_after = true) + => new GRUCell(new GRUCellArgs + { + Units = units, + Activation = keras.activations.GetActivationFromName(activation), + RecurrentActivation = keras.activations.GetActivationFromName(recurrent_activation), + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + UseBias = use_bias, + Dropout = dropout, + RecurrentDropout = recurrent_dropout, + ResetAfter = reset_after + }); + /// /// /// @@ -983,5 +1022,9 @@ namespace Tensorflow.Keras.Layers Variance = variance, Invert = invert }); + + + + } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs new file mode 100644 index 00000000..02fe54f4 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs @@ -0,0 +1,282 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Common.Extensions; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Layers.Rnn +{ + /// + /// Cell class for the GRU layer. + /// + public class GRUCell : DropoutRNNCellMixin + { + GRUCellArgs _args; + IVariableV1 _kernel; + IVariableV1 _recurrent_kernel; + IInitializer _bias_initializer; + IVariableV1 _bias; + INestStructure _state_size; + INestStructure _output_size; + int Units; + public override INestStructure StateSize => _state_size; + + public override INestStructure OutputSize => _output_size; + + public override bool SupportOptionalArgs => false; + public GRUCell(GRUCellArgs args) : base(args) + { + _args = args; + if (_args.Units <= 0) + { + throw new ValueError( + $"units must be a positive integer, got {args.Units}"); + } + _args.Dropout = Math.Min(1f, Math.Max(0f, _args.Dropout)); + _args.RecurrentDropout = Math.Min(1f, Math.Max(0f, this._args.RecurrentDropout)); + if (_args.RecurrentDropout != 0f && _args.Implementation != 1) + { + Debug.WriteLine("RNN `implementation=2` is not supported when `recurrent_dropout` is set." + + "Using `implementation=1`."); + _args.Implementation = 1; + } + Units = _args.Units; + _state_size = new NestList(Units); + _output_size = new NestNode(Units); + } + + public override void build(KerasShapesWrapper input_shape) + { + //base.build(input_shape); + + var single_shape = input_shape.ToSingleShape(); + var input_dim = single_shape[-1]; + + _kernel = add_weight("kernel", (input_dim, _args.Units * 3), + initializer: _args.KernelInitializer + ); + + _recurrent_kernel = add_weight("recurrent_kernel", (Units, Units * 3), + initializer: _args.RecurrentInitializer + ); + if (_args.UseBias) + { + Shape bias_shape; + if (!_args.ResetAfter) + { + bias_shape = new Shape(3 * Units); + } + else + { + bias_shape = (2, 3 * Units); + } + _bias = add_weight("bias", bias_shape, + initializer: _bias_initializer + ); + } + built = true; + } + + protected override Tensors Call(Tensors inputs, Tensors states = null, bool? training = null, IOptionalArgs? optional_args = null) + { + var h_tm1 = states.IsNested() ? states[0] : states.Single(); + var dp_mask = get_dropout_mask_for_cell(inputs, training.Value, count: 3); + var rec_dp_mask = get_recurrent_dropout_mask_for_cell(h_tm1, training.Value, count: 3); + + IVariableV1 input_bias = _bias; + IVariableV1 recurrent_bias = _bias; + if (_args.UseBias) + { + if (!_args.ResetAfter) + { + input_bias = _bias; + recurrent_bias = null; + } + else + { + input_bias = tf.Variable(tf.unstack(_bias.AsTensor())[0]); + recurrent_bias = tf.Variable(tf.unstack(_bias.AsTensor())[1]); + } + } + + + Tensor hh; + Tensor z; + if ( _args.Implementation == 1) + { + Tensor inputs_z; + Tensor inputs_r; + Tensor inputs_h; + if (0f < _args.Dropout && _args.Dropout < 1f) + { + inputs_z = inputs * dp_mask[0]; + inputs_r = inputs * dp_mask[1]; + inputs_h = inputs * dp_mask[2]; + } + else + { + inputs_z = inputs.Single(); + inputs_r = inputs.Single(); + inputs_h = inputs.Single(); + } + + + int startIndex = (int)_kernel.AsTensor().shape[0]; + var _kernel_slice = tf.slice(_kernel.AsTensor(), + new[] { 0, 0 }, new[] { startIndex, Units }); + var x_z = math_ops.matmul(inputs_z, _kernel_slice); + _kernel_slice = tf.slice(_kernel.AsTensor(), + new[] { 0, Units }, new[] { Units, Units}); + var x_r = math_ops.matmul( + inputs_r, _kernel_slice); + int endIndex = (int)_kernel.AsTensor().shape[1]; + _kernel_slice = tf.slice(_kernel.AsTensor(), + new[] { 0, Units * 2 }, new[] { startIndex, endIndex - Units * 2 }); + var x_h = math_ops.matmul(inputs_h, _kernel_slice); + + if(_args.UseBias) + { + x_z = tf.nn.bias_add( + x_z, tf.Variable(input_bias.AsTensor()[$":{Units}"])); + x_r = tf.nn.bias_add( + x_r, tf.Variable(input_bias.AsTensor()[$"{Units}:{Units * 2}"])); + x_h = tf.nn.bias_add( + x_h, tf.Variable(input_bias.AsTensor()[$"{Units * 2}:"])); + } + + Tensor h_tm1_z; + Tensor h_tm1_r; + Tensor h_tm1_h; + if (0f < _args.RecurrentDropout && _args.RecurrentDropout < 1f) + { + h_tm1_z = h_tm1 * rec_dp_mask[0]; + h_tm1_r = h_tm1 * rec_dp_mask[1]; + h_tm1_h = h_tm1 * rec_dp_mask[2]; + } + else + { + h_tm1_z = h_tm1; + h_tm1_r = h_tm1; + h_tm1_h = h_tm1; + } + + startIndex = (int)_recurrent_kernel.AsTensor().shape[0]; + var _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, 0 }, new[] { startIndex, Units }); + var recurrent_z = math_ops.matmul( + h_tm1_z, _recurrent_kernel_slice); + _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, Units }, new[] { startIndex, Units}); + var recurrent_r = math_ops.matmul( + h_tm1_r, _recurrent_kernel_slice); + if(_args.ResetAfter && _args.UseBias) + { + recurrent_z = tf.nn.bias_add( + recurrent_z, tf.Variable(recurrent_bias.AsTensor()[$":{Units}"])); + recurrent_r = tf.nn.bias_add( + recurrent_r, tf.Variable(recurrent_bias.AsTensor()[$"{Units}: {Units * 2}"])); + } + z = _args.RecurrentActivation.Apply(x_z + recurrent_z); + var r = _args.RecurrentActivation.Apply(x_r + recurrent_r); + + Tensor recurrent_h; + if (_args.ResetAfter) + { + endIndex = (int)_recurrent_kernel.AsTensor().shape[1]; + _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, Units * 2 }, new[] { startIndex, endIndex - Units * 2 }); + recurrent_h = math_ops.matmul( + h_tm1_h, _recurrent_kernel_slice); + if(_args.UseBias) + { + recurrent_h = tf.nn.bias_add( + recurrent_h, tf.Variable(recurrent_bias.AsTensor()[$"{Units * 2}:"])); + } + recurrent_h *= r; + } + else + { + _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, Units * 2 }, new[] { startIndex, endIndex - Units * 2 }); + recurrent_h = math_ops.matmul( + r * h_tm1_h, _recurrent_kernel_slice); + } + hh = _args.Activation.Apply(x_h + recurrent_h); + } + else + { + if (0f < _args.Dropout && _args.Dropout < 1f) + { + inputs = inputs * dp_mask[0]; + } + + var matrix_x = math_ops.matmul(inputs, _kernel.AsTensor()); + if(_args.UseBias) + { + matrix_x = tf.nn.bias_add(matrix_x, input_bias); + } + var matrix_x_spilted = tf.split(matrix_x, 3, axis: -1); + var x_z = matrix_x_spilted[0]; + var x_r = matrix_x_spilted[1]; + var x_h = matrix_x_spilted[2]; + + Tensor matrix_inner; + if (_args.ResetAfter) + { + matrix_inner = math_ops.matmul(h_tm1, _recurrent_kernel.AsTensor()); + if ( _args.UseBias) + { + matrix_inner = tf.nn.bias_add( + matrix_inner, recurrent_bias); + } + } + else + { + var startIndex = (int)_recurrent_kernel.AsTensor().shape[0]; + var _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, 0 }, new[] { startIndex, Units * 2 }); + matrix_inner = math_ops.matmul( + h_tm1, _recurrent_kernel_slice); + } + + var matrix_inner_splitted = tf.split(matrix_inner, new int[] {Units, Units, -1}, axis:-1); + var recurrent_z = matrix_inner_splitted[0]; + var recurrent_r = matrix_inner_splitted[0]; + var recurrent_h = matrix_inner_splitted[0]; + + z = _args.RecurrentActivation.Apply(x_z + recurrent_z); + var r = _args.RecurrentActivation.Apply(x_r + recurrent_r); + + if(_args.ResetAfter) + { + recurrent_h = r * recurrent_h; + } + else + { + var startIndex = (int)_recurrent_kernel.AsTensor().shape[0]; + var endIndex = (int)_recurrent_kernel.AsTensor().shape[1]; + var _recurrent_kernel_slice = tf.slice(_recurrent_kernel.AsTensor(), + new[] { 0, 2*Units }, new[] { startIndex, endIndex - 2 * Units }); + recurrent_h = math_ops.matmul( + r * h_tm1, _recurrent_kernel_slice); + } + hh = _args.Activation.Apply(x_h + recurrent_h); + } + var h = z * h_tm1 + (1 - z) * hh; + if (states.IsNested()) + { + var new_state = new NestList(h); + return new Nest(new INestStructure[] { new NestNode(h), new_state }).ToTensors(); + } + else + { + return new Nest(new INestStructure[] { new NestNode(h), new NestNode(h)}).ToTensors(); + } + + } + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 8eeee7a8..becdbcd6 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -132,5 +132,18 @@ namespace Tensorflow.Keras.UnitTest.Layers Console.WriteLine($"output: {output}"); Assert.AreEqual((5, 4), output.shape); } + + [TestMethod] + public void GRUCell() + { + var inputs = tf.random.normal((32, 10, 8)); + var rnn = tf.keras.layers.RNN(tf.keras.layers.GRUCell(4)); + var output = rnn.Apply(inputs); + Assert.AreEqual((32, 4), output.shape); + rnn = tf.keras.layers.RNN(tf.keras.layers.GRUCell(4, reset_after:false, use_bias:false)); + output = rnn.Apply(inputs); + Assert.AreEqual((32, 4), output.shape); + + } } } From 8ebe3e31e30acc5c9659146a908ccdacaf36df88 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Thu, 29 Jun 2023 22:22:21 +0800 Subject: [PATCH 32/77] fix: fix the bug of repeated progress bar in Model.fit() --- .../Keras/Engine/ICallback.cs | 3 + src/TensorFlowNET.Core/Keras/Engine/IModel.cs | 2 +- .../Callbacks/CallbackList.cs | 5 ++ .../Callbacks/Earlystopping.cs | 4 ++ src/TensorFlowNET.Keras/Callbacks/History.cs | 4 ++ .../Callbacks/ProgbarLogger.cs | 3 + .../Engine/Model.Evaluate.cs | 57 ++++++++----------- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 2 +- 8 files changed, 45 insertions(+), 35 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Engine/ICallback.cs b/src/TensorFlowNET.Core/Keras/Engine/ICallback.cs index 096dbd2e..e114ca97 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/ICallback.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/ICallback.cs @@ -14,6 +14,9 @@ public interface ICallback void on_predict_batch_end(long end_step, Dictionary logs); void on_predict_end(); void on_test_begin(); + void on_test_end(Dictionary logs); void on_test_batch_begin(long step); void on_test_batch_end(long end_step, Dictionary logs); + + } diff --git a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs index ddc72aee..19f3df9b 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/IModel.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/IModel.cs @@ -60,7 +60,7 @@ public interface IModel : ILayer bool skip_mismatch = false, object options = null); - Dictionary evaluate(Tensor x, Tensor y, + Dictionary evaluate(NDArray x, NDArray y, int batch_size = -1, int verbose = 1, int steps = -1, diff --git a/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs index 362f2280..cb16aafa 100644 --- a/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs +++ b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs @@ -73,4 +73,9 @@ public class CallbackList { callbacks.ForEach(x => x.on_test_batch_end(end_step, logs)); } + + public void on_test_end(Dictionary logs) + { + callbacks.ForEach(x => x.on_test_end(logs)); + } } diff --git a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs index 59152d9b..b3b78423 100644 --- a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs +++ b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs @@ -150,4 +150,8 @@ public class EarlyStopping: ICallback return less_op; } } + + public void on_test_end(Dictionary logs) + { + } } diff --git a/src/TensorFlowNET.Keras/Callbacks/History.cs b/src/TensorFlowNET.Keras/Callbacks/History.cs index c34f253d..6d3ff6c3 100644 --- a/src/TensorFlowNET.Keras/Callbacks/History.cs +++ b/src/TensorFlowNET.Keras/Callbacks/History.cs @@ -81,4 +81,8 @@ public class History : ICallback public void on_test_batch_end(long end_step, Dictionary logs) { } + + public void on_test_end(Dictionary logs) + { + } } diff --git a/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs index 9f2b1eb3..23b18cd4 100644 --- a/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs +++ b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs @@ -118,5 +118,8 @@ namespace Tensorflow.Keras.Callbacks } } + public void on_test_end(Dictionary logs) + { + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index eaa9eb23..c4761f87 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -27,7 +27,7 @@ namespace Tensorflow.Keras.Engine /// /// /// - public Dictionary evaluate(Tensor x, Tensor y, + public Dictionary evaluate(NDArray x, NDArray y, int batch_size = -1, int verbose = 1, int steps = -1, @@ -115,62 +115,53 @@ namespace Tensorflow.Keras.Engine /// The function to be called on each batch of data. /// Whether it is validation or test. /// - Dictionary evaluate(DataHandler data_handler, CallbackList callbacks, bool is_val, Func> test_func) + Dictionary evaluate(DataHandler data_handler, CallbackList callbacks, bool is_val, Func> test_func) { callbacks.on_test_begin(); - var results = new Dictionary(); - var logs = results; + var logs = new Dictionary(); foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { reset_metrics(); - callbacks.on_epoch_begin(epoch); - // data_handler.catch_stop_iteration(); - foreach (var step in data_handler.steps()) { callbacks.on_test_batch_begin(step); - - logs = test_func(data_handler, iterator.next()); - - tf_with(ops.control_dependencies(Array.Empty()), ctl => _train_counter.assign_add(1)); - + logs = test_func(data_handler, iterator); var end_step = step + data_handler.StepIncrement; if (!is_val) callbacks.on_test_batch_end(end_step, logs); } - - if (!is_val) - callbacks.on_epoch_end(epoch, logs); } - - foreach (var log in logs) - { - results[log.Key] = log.Value; - } - + callbacks.on_test_end(logs); + var results = new Dictionary(logs); return results; } - Dictionary test_function(DataHandler data_handler, Tensor[] data) + Dictionary test_function(DataHandler data_handler, OwnedIterator iterator) { - var (x, y) = data_handler.DataAdapter.Expand1d(data[0], data[1]); - - var y_pred = Apply(x, training: false); - var loss = compiled_loss.Call(y, y_pred); - - compiled_metrics.update_state(y, y_pred); - - var outputs = metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Name, x => (float)x.Item2); + var data = iterator.next(); + var outputs = test_step(data_handler, data[0], data[1]); + tf_with(ops.control_dependencies(new object[0]), ctl => _test_counter.assign_add(1)); return outputs; } - Dictionary test_step_multi_inputs_function(DataHandler data_handler, Tensor[] data) + Dictionary test_step_multi_inputs_function(DataHandler data_handler, OwnedIterator iterator) { + var data = iterator.next(); var x_size = data_handler.DataAdapter.GetDataset().FirstInputTensorCount; - var outputs = train_step(data_handler, new Tensors(data.Take(x_size).ToArray()), new Tensors(data.Skip(x_size).ToArray())); - tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); + var outputs = test_step(data_handler, data.Take(x_size).ToArray(), data.Skip(x_size).ToArray()); + tf_with(ops.control_dependencies(new object[0]), ctl => _test_counter.assign_add(1)); return outputs; } + + + Dictionary test_step(DataHandler data_handler, Tensors x, Tensors y) + { + (x, y) = data_handler.DataAdapter.Expand1d(x, y); + var y_pred = Apply(x, training: false); + var loss = compiled_loss.Call(y, y_pred); + compiled_metrics.update_state(y, y_pred); + return metrics.Select(x => (x.Name, x.result())).ToDictionary(x => x.Item1, x => (float)x.Item2); + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index 68dc5976..76c592ad 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -266,7 +266,7 @@ namespace Tensorflow.Keras.Engine { // Because evaluate calls call_test_batch_end, this interferes with our output on the screen // so we need to pass a is_val parameter to stop on_test_batch_end - var val_logs = evaluate((Tensor)validation_data.Value.Item1, validation_data.Value.Item2, is_val:true); + var val_logs = evaluate(validation_data.Value.Item1, validation_data.Value.Item2, is_val:true); foreach (var log in val_logs) { logs["val_" + log.Key] = log.Value; From 4e0c3df4a40bbb3fce8bbc6f68a885391350af8d Mon Sep 17 00:00:00 2001 From: RachamimYaakobov Date: Fri, 30 Jun 2023 16:16:00 +0300 Subject: [PATCH 33/77] Bug fix in KerasObjectLoader.cs I added `ToArray()` so that there is no "The collection has changed" error after `_delete_tracking`. --- src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index 396ad20e..1e869d66 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -174,7 +174,7 @@ namespace Tensorflow.Keras.Saving } if(node is Functional functional) { - foreach(var name in functional.UnconditionalDependencyNames.Keys) + foreach(var name in functional.UnconditionalDependencyNames.Keys.ToArray()) { if(Regex.Match(name, @"^layer(_with_weights)?-[\d+]").Success) { From f61ab520c91de2b25bf09356735b9617278f5a44 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Fri, 30 Jun 2023 21:25:35 +0800 Subject: [PATCH 34/77] fix inconsistent shape error while training Embedding layer. --- src/TensorFlowNET.Core/Framework/IndexedSlices.cs | 15 ++++++++++++++- .../Layers/LayersTest.cs | 11 +++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Framework/IndexedSlices.cs b/src/TensorFlowNET.Core/Framework/IndexedSlices.cs index 24d356fb..bac5e6fb 100644 --- a/src/TensorFlowNET.Core/Framework/IndexedSlices.cs +++ b/src/TensorFlowNET.Core/Framework/IndexedSlices.cs @@ -49,12 +49,25 @@ namespace Tensorflow.Framework public static implicit operator Tensor(IndexedSlices indexedSlices) { - return indexedSlices.values; + return _indexed_slices_to_tensor(indexedSlices); } public static implicit operator IndexedSlices(Tensor tensor) { return tensor.Tag as IndexedSlices; } + + /// + /// Converts an IndexedSlices object `value` to a Tensor. + /// + /// + /// + /// + /// + /// + public static Tensor _indexed_slices_to_tensor(IndexedSlices indexedSlices, TF_DataType dtype = TF_DataType.DtInvalid, String name = "", bool as_ref = false) + { + return gen_math_ops.unsorted_segment_sum(indexedSlices.values, indexedSlices.indices, indexedSlices.dense_shape.slice(0)); + } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index 98d90966..7ebb53db 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -110,6 +110,17 @@ namespace Tensorflow.Keras.UnitTest.Layers var output_array = model.predict(input_array); Assert.AreEqual((32, 10, 64), output_array.shape); } + [TestMethod] + public void EmbeddingGrad() + { + var inputs = keras.layers.Input(shape: new[] { 32, 10 }); + var outputs = keras.layers.Embedding(1000, 64, input_length: 10).Apply(inputs); + var model = keras.Model(inputs: inputs, outputs: outputs); + var input_array = np.random.randint(1000, size: (1, 32, 10)); + var output_array = np.random.random(size: (1, 32, 10, 64)); + model.compile("rmsprop", "mse", new[] { "accuracy" }); + model.fit(input_array, output_array); + } /// /// https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense From 3acfc1dcb0bb978e69184858712765c39c03ef0c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 30 Jun 2023 12:50:16 -0500 Subject: [PATCH 35/77] tf.math.reduce_euclidean_norm --- src/TensorFlowNET.Core/APIs/tf.math.cs | 11 +++++++++ src/TensorFlowNET.Core/Operations/math_ops.cs | 11 +++++++++ .../ManagedAPI/MathApiTest.cs | 23 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index ffbc4373..c999933c 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -46,6 +46,17 @@ namespace Tensorflow public Tensor divide_no_nan(Tensor a, Tensor b, string name = null) => math_ops.div_no_nan(a, b); + /// + /// Computes the Euclidean norm of elements across dimensions of a tensor. + /// + /// The tensor to reduce. Should have numeric type. + /// The dimensions to reduce. If `None` (the default), reduces all dimensions.Must be in the range `[-rank(input_tensor), rank(input_tensor))` + /// If true, retains reduced dimensions with length 1. + /// A name for the operation (optional). + /// The reduced tensor, of the same dtype as the input_tensor. + public Tensor reduce_euclidean_norm(Tensor input_tensor, Axis? axis = null, bool keepdims = false, string name = null) + => math_ops.reduce_euclidean_norm(input_tensor, axis: axis, keepdims: keepdims, name); + public Tensor square(Tensor x, string name = null) => math_ops.square(x, name: name); diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index d00a5d36..6d386052 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -587,6 +587,17 @@ namespace Tensorflow return _may_reduce_to_scalar(keepdims, axis, max); } + public static Tensor reduce_euclidean_norm(Tensor input_tensor, Axis axis = null, bool keepdims = false, string name = null) + { + var r = _ReductionDims(input_tensor, axis); + var distance = tf.Context.ExecuteOp("EuclideanNorm", name, + new ExecuteOpArgs(input_tensor, r).SetAttributes(new + { + keep_dims = keepdims + })); + return _may_reduce_to_scalar(keepdims, axis, distance); + } + public static Tensor reduce_max(Tensor input_tensor, Axis axis = null, bool keepdims = false, string name = null) { var r = _ReductionDims(input_tensor, axis); diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/MathApiTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/MathApiTest.cs index 42ac641b..411deb18 100644 --- a/test/TensorFlowNET.UnitTest/ManagedAPI/MathApiTest.cs +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/MathApiTest.cs @@ -1,6 +1,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using System.Linq; using Tensorflow; +using Tensorflow.NumPy; using static Tensorflow.Binding; namespace TensorFlowNET.UnitTest.ManagedAPI @@ -57,5 +59,26 @@ namespace TensorFlowNET.UnitTest.ManagedAPI var actual = erf.ToArray(); Assert.IsTrue(Equal(expected, actual)); } + + [TestMethod] + public void ReduceEuclideanNorm() + { + var x = tf.constant(new[,] { { 1, 2, 3 }, { 1, 1, 1 } }); + Assert.AreEqual(tf.math.reduce_euclidean_norm(x).numpy(), 4); + + var y = tf.constant(new[,] { { 1, 2, 3 }, { 1, 1, 1 } }, dtype: tf.float32); + Assert.IsTrue(Equal(tf.math.reduce_euclidean_norm(y).numpy(), 4.1231055f)); + + Assert.IsTrue(Equal(tf.math.reduce_euclidean_norm(y, 0).ToArray(), + new float[] { np.sqrt(2f), np.sqrt(5f), np.sqrt(10f) })); + + Assert.IsTrue(Equal(tf.math.reduce_euclidean_norm(y, 1).ToArray(), + new float[] { np.sqrt(14f), np.sqrt(3f) })); + + Assert.IsTrue(Equal(tf.math.reduce_euclidean_norm(y, 1, keepdims: true).ToArray(), + new float[] { np.sqrt(14f), np.sqrt(3f) })); + + Assert.AreEqual(tf.math.reduce_euclidean_norm(y, (0, 1)).numpy(), np.sqrt(17f)); + } } } From 4efa0a8881a5886a2eeb2e19d8a5157c3f68a32f Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Sat, 1 Jul 2023 13:44:54 +0800 Subject: [PATCH 36/77] add pad preprocessing for `imdb` dataset --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 56b0d2a7..61ce3947 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -40,6 +40,8 @@ namespace Tensorflow.Keras.Datasets int oov_char= 2, int index_from = 3) { + if (maxlen == -1) throw new InvalidArgumentError("maxlen must be assigned."); + var dst = Download(); var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); @@ -51,7 +53,7 @@ namespace Tensorflow.Keras.Datasets x_train_string[i] = lines[i].Substring(2); } - var x_train = np.array(x_train_string); + var x_train = keras.preprocessing.sequence.pad_sequences(PraseData(x_train_string), maxlen: maxlen); File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); var x_test_string = new string[lines.Length]; @@ -62,7 +64,7 @@ namespace Tensorflow.Keras.Datasets x_test_string[i] = lines[i].Substring(2); } - var x_test = np.array(x_test_string); + var x_test = keras.preprocessing.sequence.pad_sequences(PraseData(x_test_string), maxlen: maxlen); return new DatasetPass { @@ -93,5 +95,23 @@ namespace Tensorflow.Keras.Datasets return dst; // return Path.Combine(dst, file_name); } + + protected IEnumerable PraseData(string[] x) + { + var data_list = new List(); + for (int i = 0; i < len(x); i++) + { + var list_string = x[i]; + var cleaned_list_string = list_string.Replace("[", "").Replace("]", "").Replace(" ", ""); + string[] number_strings = cleaned_list_string.Split(','); + int[] numbers = new int[number_strings.Length]; + for (int j = 0; j < number_strings.Length; j++) + { + numbers[j] = int.Parse(number_strings[j]); + } + data_list.Add(numbers); + } + return data_list; + } } } From a76cd67d3060aabb8f658fc11146c1dc9bccaa0c Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Mon, 3 Jul 2023 13:26:45 +0000 Subject: [PATCH 37/77] fix some api's bug --- src/TensorFlowNET.Core/APIs/tf.nn.cs | 12 ++---------- src/TensorFlowNET.Core/Operations/array_ops.cs | 1 - 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.nn.cs b/src/TensorFlowNET.Core/APIs/tf.nn.cs index e5cd4e56..397c68c7 100644 --- a/src/TensorFlowNET.Core/APIs/tf.nn.cs +++ b/src/TensorFlowNET.Core/APIs/tf.nn.cs @@ -144,16 +144,8 @@ namespace Tensorflow Tensor offset, Tensor scale, float variance_epsilon, - string name = null) - { - var inv = math_ops.rsqrt(variance + variance_epsilon); - tf_with(ops.name_scope(name, "batchnorm", (x, mean, variance, scale, offset)), scope => - { - if (scale != null) inv *= scale; - }); - if (offset != null) return x * math_ops.cast(inv, x.dtype) + math_ops.cast(offset - mean * inv, dtype: x.dtype); - else return x * math_ops.cast(inv, x.dtype) + math_ops.cast(-mean * inv, dtype: x.dtype); - } + string name = null) => nn_impl.batch_normalization(x, mean, variance, offset, scale, variance_epsilon, name); + public Tensor max_pool(Tensor value, int[] ksize, int[] strides, string padding, string data_format = "NHWC", string name = null) => nn_ops.max_pool(value, ksize, strides, padding, data_format: data_format, name: name); diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 7f787533..fbb3bf11 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -678,7 +678,6 @@ namespace Tensorflow var tape = tf.GradientTape().stop_recording(); var result = gen_array_ops.stop_gradient(input, name); tape.StartRecord(); - tf.GradientTape().PushTape(tape); return result; } From f026963a7da0cf8444f05fd4abce8962b5848c62 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Wed, 5 Jul 2023 21:46:50 +0800 Subject: [PATCH 38/77] add EinsumGrad --- src/TensorFlowNET.Core/Gradients/math_grad.cs | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/src/TensorFlowNET.Core/Gradients/math_grad.cs b/src/TensorFlowNET.Core/Gradients/math_grad.cs index be1fbbba..8c3f0f8b 100644 --- a/src/TensorFlowNET.Core/Gradients/math_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/math_grad.cs @@ -117,6 +117,137 @@ namespace Tensorflow.Gradients }; } + public static string ellipsis = "..."; + [RegisterGradient("Einsum")] + public static Tensor[] _EinsumGrad(Operation op, Tensor[] grads) + { + // Gradient for Einsum. + string equation = (string)op.get_attr("equation"); + string[] split_equation = equation.Split(new string[] { "->" }, StringSplitOptions.None); + var input_subs = split_equation[0]; + var output_subs = split_equation[1]; + + if (op.inputs.Length == 1) + { + var input_shape = array_ops.shape(op.inputs[0]); + var reduced_label_set = new HashSet(new HashSet(input_subs).Except(new HashSet(output_subs + ellipsis))); + if (reduced_label_set.Count == 0) + return new Tensor[] { math_ops.einsum(string.Format("{0}->{1}", output_subs, input_subs), new Tensors(grads)) }; + return new Tensor[] { _GetGradReduced(new Tensors(grads), output_subs, input_subs, input_shape, reduced_label_set) }; + } + + string[] split_input_subs = input_subs.Split(new string[] { "," }, StringSplitOptions.None); + var x_subs = split_input_subs[0]; + var y_subs = split_input_subs[1]; + // Add ellipsis for broadcasted dimensions if any operand does not have it. + // This is because the equation "...ij,jk->ik" may be valid if the 0th input's + // batch shape is empty, but the VJP equation "jk,ik->...ij" is not valid + // because only the output subscripts contain ellipsis. + if (output_subs.Contains(ellipsis)) + { + if (!x_subs.Contains(ellipsis)) + x_subs += ellipsis; + if (!y_subs.Contains(ellipsis)) + y_subs += ellipsis; + } + // Obtain the gradients wrt the inputs x and y, without taking into account + // the unbroadcasting. + var x = op.inputs[0]; + var y = op.inputs[1]; + if (grads.GetDataType().is_complex()) + { + x = math_ops.conj(x); + y = math_ops.conj(y); + } + + var x_shape = array_ops.shape(x); + var y_shape = array_ops.shape(y); + var grad_x = _GetGradWrt(grads, y, x_shape, x_subs, y_subs, output_subs); + var grad_y = _GetGradWrt(grads, x, y_shape, y_subs, x_subs, output_subs); + + if (!output_subs.Contains(ellipsis)) + return new Tensor[] { grad_x, grad_y }; + var bx = _GetBcastSubshape(x_subs); + int bx_start = bx[0], bx_end = bx[1]; + var by = _GetBcastSubshape(y_subs); + int by_start = by[0], by_end = by[1]; + + var x_shape_static = x.shape; + var y_shape_static = y.shape; + if(x_shape_static.IsFullyDefined && + y_shape_static.IsFullyDefined && + x_shape_static[string.Format("{0}:{1}",bx_start,bx_end)] == y_shape_static[string.Format("{0}:{1}", by_start, by_end)]) + return new Tensor[] { grad_x, grad_y }; + + var r = gen_array_ops.broadcast_gradient_args(x_shape[string.Format("{0}:{1}", bx_start, bx_end)], + y_shape[string.Format("{0}:{1}", by_start, by_end)]); + var rx = r[0]; + var ry = r[1]; + grad_x = array_ops.reshape(math_ops.reduce_sum(grad_x, bx_start + rx), x_shape); + grad_y = array_ops.reshape(math_ops.reduce_sum(grad_y, by_start + ry), y_shape); + return new Tensor[] { grad_x, grad_y }; + } + protected static Tensor _GetGradWrt(Tensor[] output_grads, Tensor other_operand, Tensor input_shape, + string input_subs, string other_subs, string output_subs) + { + var reduced_label_set = new HashSet(new HashSet(input_subs).Except(new HashSet(output_subs + other_subs + "."))); + var left_subs = string.Join("", input_subs.Where(s => !reduced_label_set.Contains(s))); + var grad_reduced = math_ops.einsum(string.Format("{0},{1}->{2}", output_subs, other_subs, left_subs), new Tensors((Tensors)output_grads, other_operand)); + if (reduced_label_set.Count == 0) + return grad_reduced; + return _GetGradReduced(grad_reduced, left_subs, input_subs, input_shape, reduced_label_set); + } + protected static Tensor _GetGradReduced(Tensor output_grad, string output_subs, string input_subs, Tensor input_shape, HashSet reduced_label_set) + { + string reduced_subs; + Tensor reduced_dims; + List reduced_axes; + _GetReducedSubscripts(reduced_label_set, input_shape, input_subs, out reduced_subs, out reduced_dims, out reduced_axes); + bool has_repeated_labels = ( + new HashSet(input_subs).Count + new HashSet(output_subs).Count < + input_subs.Length + output_subs.Length); + var input_subs_without_reduced_labels = string.Join("", input_subs.Where(s => !reduced_label_set.Contains(s))); + + if (!has_repeated_labels && input_subs_without_reduced_labels == output_subs) + { + var reduced_shape = math_ops.reduced_shape(input_shape, ops.convert_to_tensor(reduced_axes)); + return gen_array_ops.broadcast_to(array_ops.reshape(output_grad, reduced_shape), input_shape); + } + else + { + var grad_shape_with_reduced_labels = array_ops.concat(new Tensor[] { reduced_dims, array_ops.shape(new Tensors(output_grad)) }, axis: 0); + var reduced_shape = array_ops.concat(new Tensor[] { array_ops.ones(reduced_label_set.Count, dtype: dtypes.int32), array_ops.shape(new Tensors(output_grad)) }, axis: 0); + var broadcasted_grad = gen_array_ops.broadcast_to(array_ops.reshape(output_grad, reduced_shape), grad_shape_with_reduced_labels); + return math_ops.einsum(string.Format("{0}->{1}", reduced_subs + output_subs, input_subs), new Tensors(broadcasted_grad)); + } + } + protected static void _GetReducedSubscripts(HashSet reduced_label_set, Tensor input_shape, string subscripts, out string reduced_subs, out Tensor reduced_dims, out List reduced_axes) + { + reduced_subs = string.Join("", reduced_label_set.Select(c => c.ToString())); + reduced_axes = reduced_subs.Select(s => _GetAxisFromLabel(subscripts, s)).ToList(); + reduced_dims = array_ops.stack(reduced_axes.Select(ax => input_shape[ax]).ToList()); + } + protected static int _GetAxisFromLabel(string subscripts, char label) + { + var splits = subscripts.Split(new string[] { ellipsis }, StringSplitOptions.None); + var index = splits[0].IndexOf(label); + if (index != -1) return index; + if (splits.Length < 2) throw new OutOfRangeError(); + index = splits[1].IndexOf(label); + if (index != -1) return index; + throw new ValueError(); + } + protected static int[] _GetBcastSubshape(string subscripts) + { + int start = subscripts.IndexOf(ellipsis); + if (start == -1) return new int[] { 0, 0 }; + int remaining = subscripts.Length - (start + ellipsis.Length); + int end; + if (remaining > 0) end = remaining; + else throw new Exception(); + return new int[] { start, end }; + } + /// /// Returns grad * exp(x). /// From 6862d3a0432b0623bba23e51c14d42ac1974e22f Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Fri, 7 Jul 2023 00:25:38 +0000 Subject: [PATCH 39/77] Add AdamW optimizer --- src/TensorFlowNET.Core/Keras/IOptimizerApi.cs | 21 ++++++ src/TensorFlowNET.Keras/Optimizers/AdamW.cs | 67 +++++++++++++++++++ .../Optimizers/OptimizerApi.cs | 16 +++++ 3 files changed, 104 insertions(+) create mode 100644 src/TensorFlowNET.Keras/Optimizers/AdamW.cs diff --git a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs index 961ce91a..d0d3a74f 100644 --- a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs +++ b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs @@ -25,6 +25,27 @@ namespace Tensorflow.Keras bool amsgrad = false, string name = "Adam"); + /// + /// Adam enables L2 weight decay on gradients. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + IOptimizer AdamW(float learning_rate = 0.001f, + float weight_decay = 0.004f, + float beta_1 = 0.9f, + float beta_2 = 0.999f, + float epsilon = 1e-7f, + bool amsgrad = false, + List no_decay_params = null, + string name = "AdamW"); + /// /// Construct a new RMSprop optimizer. /// diff --git a/src/TensorFlowNET.Keras/Optimizers/AdamW.cs b/src/TensorFlowNET.Keras/Optimizers/AdamW.cs new file mode 100644 index 00000000..469b8ad2 --- /dev/null +++ b/src/TensorFlowNET.Keras/Optimizers/AdamW.cs @@ -0,0 +1,67 @@ +namespace Tensorflow.Keras.Optimizers +{ + public class AdamW : Adam + { + string name; + float weight_decay; + DeviceDType deType; + List no_decay_params = null; + public AdamW(float learning_rate= 0.001f, + float weight_decay= 0.004f, + float beta_1= 0.9f, + float beta_2= 0.999f, + float epsilon= 1e-7f, + bool amsgrad = false, + List no_decay_params = null, + string name= "AdamW") : base(learning_rate, beta_1, beta_2, epsilon, amsgrad) + { + this.name = name; + this.weight_decay = weight_decay; + this.no_decay_params = no_decay_params; + } + + protected Operation _decay_weights_op(IVariableV1 var, float learning_rate, Dictionary> apply_state) + { + var device_dtype = new DeviceDType(); + device_dtype.DType = var.dtype; + device_dtype.Device = var.Device; + bool do_decay = _do_use_weight_decay(var.Name); + if (do_decay) return var.assign_add( + -learning_rate * var.AsTensor() * apply_state[deType]["weight_decay"]); + return tf.no_op(); + } + + + protected bool _do_use_weight_decay(string param_name) + { + // Whether to use L2 weight decay for `param_name`. + if (this.weight_decay == 0) + return false; + + if (this.no_decay_params != null) + { + foreach (var name in no_decay_params) + { + if (param_name.Contains(name)) return false; + } + + } + return true; + } + + protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, Dictionary> apply_state) + { + var decay = _decay_weights_op(var, _hyper["learning_rate"], apply_state); + tf.control_dependencies(new[] { decay }); + return base._resource_apply_dense(var, grad, apply_state); + } + + protected override void _prepare_local(DeviceDType device_dtype, Dictionary> apply_state) + { + this.deType = device_dtype; + base._prepare_local(device_dtype, apply_state); + apply_state[device_dtype]["weight_decay"] = tf.constant( + weight_decay, name: "adam_weight_decay_rate"); + } + } +} diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs index 31eb88be..28069426 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs @@ -29,6 +29,22 @@ namespace Tensorflow.Keras.Optimizers amsgrad: amsgrad, name: name); + public IOptimizer AdamW(float learning_rate = 0.001f, + float weight_decay = 0.004f, + float beta_1 = 0.9f, + float beta_2 = 0.999f, + float epsilon = 1e-7f, + bool amsgrad = false, + List no_decay_params = null, + string name = "AdamW") => new AdamW(learning_rate: learning_rate, + beta_1: beta_1, + beta_2: beta_2, + epsilon: epsilon, + amsgrad: amsgrad, + name: name, + weight_decay: weight_decay, + no_decay_params: no_decay_params); + /// /// Construct a new RMSprop optimizer. /// From cc6ddc144fa85010b111df2b4c596c7230052080 Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Fri, 7 Jul 2023 00:33:41 +0000 Subject: [PATCH 40/77] Add AdamW optimizer --- src/TensorFlowNET.Keras/Optimizers/AdamW.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Keras/Optimizers/AdamW.cs b/src/TensorFlowNET.Keras/Optimizers/AdamW.cs index 469b8ad2..d111b5d3 100644 --- a/src/TensorFlowNET.Keras/Optimizers/AdamW.cs +++ b/src/TensorFlowNET.Keras/Optimizers/AdamW.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.Optimizers +namespace Tensorflow.Keras.Optimizers { public class AdamW : Adam { @@ -22,9 +22,6 @@ protected Operation _decay_weights_op(IVariableV1 var, float learning_rate, Dictionary> apply_state) { - var device_dtype = new DeviceDType(); - device_dtype.DType = var.dtype; - device_dtype.Device = var.Device; bool do_decay = _do_use_weight_decay(var.Name); if (do_decay) return var.assign_add( -learning_rate * var.AsTensor() * apply_state[deType]["weight_decay"]); From 42e8b046ea53f3173ac92fe7cbc1d57d3d796fa8 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 7 Jul 2023 21:25:41 -0500 Subject: [PATCH 41/77] Release v0.110.1. --- src/TensorFlowNET.Core/APIs/tf.array.cs | 3 +-- src/TensorFlowNET.Core/Operations/array_ops.cs | 16 +--------------- src/TensorFlowNET.Core/Operations/math_ops.cs | 11 +++-------- src/TensorFlowNET.Core/Tensorflow.Binding.csproj | 6 +++--- src/TensorFlowNET.Core/ops.cs | 8 +++++++- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 6 +++--- 6 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs index 6a646512..ecac37eb 100644 --- a/src/TensorFlowNET.Core/APIs/tf.array.cs +++ b/src/TensorFlowNET.Core/APIs/tf.array.cs @@ -91,8 +91,7 @@ namespace Tensorflow return identity(values.First(), name: scope); }); } - - return gen_array_ops.concat_v2(values.ToArray(), ops.convert_to_tensor(axis), name: name); + return array_ops.concat(values.ToArray(), axis, name: name); } /// diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index fbb3bf11..5237ec44 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -892,23 +892,9 @@ namespace Tensorflow /// /// /// - public static Tensor concat(Tensor[] values, int axis, string name = "concat") - { - if (values.Length == 1) // Degenerate case of one tensor. - { - return tf_with(ops.name_scope(name), scope => - { - var t = ops.convert_to_tensor(axis, name: "concat_dim", dtype: TF_DataType.TF_INT32); - return identity(values[0], name: scope); - }); - } - - return gen_array_ops.concat_v2(values, ops.convert_to_tensor(axis), name: name); - } - public static Tensor concat(Tensor[] values, Tensor axis, string name = "concat") { - return gen_array_ops.concat_v2(values, axis, name: name); + return tf.Context.ExecuteOp("ConcatV2", name, new ExecuteOpArgs(values, axis)); } public static Tensor concat(object[] values, int axis, string name = "concat") diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 6d386052..092137bf 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -791,10 +791,7 @@ namespace Tensorflow bool adjoint_a = false, bool adjoint_b = false, bool a_is_sparse = false, bool b_is_sparse = false, string name = null) - { - Tensor result = null; - - tf_with(ops.name_scope(name, "MatMul", new Tensor[] { a, b }), scope => + => tf_with(ops.name_scope(name, "MatMul", (a, b)), scope => { name = scope; @@ -815,12 +812,10 @@ namespace Tensorflow transpose_b = true; } - result = gen_math_ops.mat_mul(a, b, transpose_a, transpose_b, name); + return tf.Context.ExecuteOp("MatMul", name, new ExecuteOpArgs(a, b) + .SetAttributes(new { transpose_a, transpose_b })); }); - return result; - } - public static Tensor batch_matmul(Tensor x, Tensor y, bool adj_x = false, bool adj_y = false, string name = null) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 3bc20289..6a2dcff7 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.10.0 - 0.110.0 + 0.110.1 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.110.0.0 + 0.110.1.0 tf.net 0.110.x and above are based on tensorflow native 2.11.0 * RNN, LSTM works. @@ -42,7 +42,7 @@ https://tensorflownet.readthedocs.io tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. - 0.110.0.0 + 0.110.1.0 LICENSE true packages diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 7bd78a79..2dc46329 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -138,9 +138,15 @@ namespace Tensorflow else { var graph = get_default_graph(); + if (graph is FuncGraph funcGraph) + { + return funcGraph.capture(eager_tensor, name: name); + } if (!graph.building_function) + { throw new RuntimeError("Attempting to capture an EagerTensor without building a function."); - return (graph as FuncGraph).capture(eager_tensor, name: name); + // return eager_tensor.AsPlaceholder(name: name); + } } } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 5dc46fe4..ab667519 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.0 + 0.11.1 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2023 @@ -38,8 +38,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.11.0.0 - 0.11.0.0 + 0.11.1.0 + 0.11.1.0 LICENSE Debug;Release;GPU From 992bf55dab0273de568e8347d29fdc19e3ad4aa0 Mon Sep 17 00:00:00 2001 From: Beacontownfc <19636977267@qq.com> Date: Sat, 8 Jul 2023 02:39:06 +0000 Subject: [PATCH 42/77] fix load_weights --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index b04391be..8ac9fddf 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -7,6 +7,8 @@ using HDF5CSharp; using static Tensorflow.Binding; using static Tensorflow.KerasApi; using System.Linq; +using System.Text.RegularExpressions; + namespace Tensorflow.Keras.Saving { public class hdf5_format @@ -132,7 +134,9 @@ namespace Tensorflow.Keras.Saving var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); foreach (var i_ in weight_names) { - (success, Array result) = Hdf5.ReadDataset(g, i_); + var vm = Regex.Replace(i_, "/", "$"); + vm = i_.Split('/')[0] + "/$" + vm.Substring(i_.Split('/')[0].Length + 1, i_.Length - i_.Split('/')[0].Length - 1); + (success, Array result) = Hdf5.ReadDataset(g, vm); if (success) weight_values.Add(np.array(result)); } @@ -193,7 +197,8 @@ namespace Tensorflow.Keras.Saving if (name.IndexOf("/") > 1) { var crDataGroup = Hdf5.CreateOrOpenGroup(g, Hdf5Utils.NormalizedName(name.Split('/')[0])); - WriteDataset(crDataGroup, name.Split('/')[1], tensor); + var _name = Regex.Replace(name.Substring(name.Split('/')[0].Length, name.Length - name.Split('/')[0].Length), "/", "$"); + WriteDataset(crDataGroup, _name, tensor); Hdf5.CloseGroup(crDataGroup); } else From f01558b642cc7719ac19296374cb897f337300cf Mon Sep 17 00:00:00 2001 From: BalashovK Date: Sat, 8 Jul 2023 15:39:08 -0700 Subject: [PATCH 43/77] exp moved to tf.math.cs --- src/TensorFlowNET.Core/APIs/tf.exp.cs | 25 ------------------------- src/TensorFlowNET.Core/APIs/tf.math.cs | 2 ++ 2 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 src/TensorFlowNET.Core/APIs/tf.exp.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.exp.cs b/src/TensorFlowNET.Core/APIs/tf.exp.cs deleted file mode 100644 index 56ea1898..00000000 --- a/src/TensorFlowNET.Core/APIs/tf.exp.cs +++ /dev/null @@ -1,25 +0,0 @@ -/***************************************************************************** - Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -namespace Tensorflow -{ - public partial class tensorflow - { - public Tensor exp(Tensor x, - string name = null) => gen_math_ops.exp(x, name); - - } -} diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index c999933c..da54a9dd 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -622,5 +622,7 @@ namespace Tensorflow => gen_math_ops.squared_difference(x: x, y: y, name: name); public Tensor complex(Tensor real, Tensor imag, Tensorflow.TF_DataType? dtype = null, string name = null) => gen_ops.complex(real, imag, dtype, name); + public Tensor exp(Tensor x, + string name = null) => gen_math_ops.exp(x, name); } } From 4b8e63bb8213ca969b7d03ff3aa76c189f5c1b99 Mon Sep 17 00:00:00 2001 From: BalashovK Date: Sat, 8 Jul 2023 15:39:08 -0700 Subject: [PATCH 44/77] fix: exp moved to tf.math.cs --- src/TensorFlowNET.Core/APIs/tf.exp.cs | 25 ------------------------- src/TensorFlowNET.Core/APIs/tf.math.cs | 2 ++ 2 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 src/TensorFlowNET.Core/APIs/tf.exp.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.exp.cs b/src/TensorFlowNET.Core/APIs/tf.exp.cs deleted file mode 100644 index 56ea1898..00000000 --- a/src/TensorFlowNET.Core/APIs/tf.exp.cs +++ /dev/null @@ -1,25 +0,0 @@ -/***************************************************************************** - Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -******************************************************************************/ - -namespace Tensorflow -{ - public partial class tensorflow - { - public Tensor exp(Tensor x, - string name = null) => gen_math_ops.exp(x, name); - - } -} diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index c999933c..da54a9dd 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -622,5 +622,7 @@ namespace Tensorflow => gen_math_ops.squared_difference(x: x, y: y, name: name); public Tensor complex(Tensor real, Tensor imag, Tensorflow.TF_DataType? dtype = null, string name = null) => gen_ops.complex(real, imag, dtype, name); + public Tensor exp(Tensor x, + string name = null) => gen_math_ops.exp(x, name); } } From b968fd79ab156bfca62f434c7fb936e2ed512455 Mon Sep 17 00:00:00 2001 From: dogvane Date: Mon, 10 Jul 2023 00:41:23 +0800 Subject: [PATCH 45/77] add avg_pool_grad function --- src/TensorFlowNET.Core/Gradients/nn_grad.cs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/TensorFlowNET.Core/Gradients/nn_grad.cs b/src/TensorFlowNET.Core/Gradients/nn_grad.cs index a1ac97a9..3a6efd54 100644 --- a/src/TensorFlowNET.Core/Gradients/nn_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/nn_grad.cs @@ -365,6 +365,23 @@ namespace Tensorflow.Gradients }; } + [RegisterGradient("AvgPool")] + public static Tensor[] _AvgPoolGrad(Operation op, Tensor[] grads) + { + Tensor grad = grads[0]; + + return new Tensor[] + { + gen_nn_ops.avg_pool_grad( + array_ops.shape(op.inputs[0]), + grad, + op.get_attr_list("ksize"), + op.get_attr_list("strides"), + op.get_attr("padding").ToString(), + op.get_attr("data_format").ToString()) + }; + } + /// /// Return the gradients for TopK. /// From fa213eb54c2b3c1b28d9ca4ebc2a49d90a0e46bf Mon Sep 17 00:00:00 2001 From: dogvane Date: Mon, 10 Jul 2023 00:52:15 +0800 Subject: [PATCH 46/77] change "bool training" => "bool? training" the bool to tensor has a bug, if in init the training is False, the program not start. --- src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 2 +- src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs | 2 +- src/TensorFlowNET.Keras/Engine/Layer.Apply.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 8 ++++++-- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index e94c8bf1..2f92c4e5 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -15,7 +15,7 @@ namespace Tensorflow.Keras List Layers { get; } List InboundNodes { get; } List OutboundNodes { get; } - Tensors Apply(Tensors inputs, Tensors states = null, bool training = false, IOptionalArgs? optional_args = null); + Tensors Apply(Tensors inputs, Tensors states = null, bool? training = false, IOptionalArgs? optional_args = null); List TrainableVariables { get; } List TrainableWeights { get; } List NonTrainableWeights { get; } diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index e488c47e..4e99731f 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -145,7 +145,7 @@ namespace Tensorflow throw new NotImplementedException("_zero_state_tensors"); } - public Tensors Apply(Tensors inputs, Tensors state = null, bool is_training = false, IOptionalArgs? optional_args = null) + public Tensors Apply(Tensors inputs, Tensors state = null, bool? is_training = false, IOptionalArgs? optional_args = null) { throw new NotImplementedException(); } diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index d52190fd..8a66948b 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -13,7 +13,7 @@ namespace Tensorflow.Keras.Engine /// /// /// - public virtual Tensors Apply(Tensors inputs, Tensors states = null, bool training = false, IOptionalArgs? optional_args = null) + public virtual Tensors Apply(Tensors inputs, Tensors states = null, bool? training = false, IOptionalArgs? optional_args = null) { if (callContext.Value == null) callContext.Value = new CallContext(); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index 76c592ad..de57f19a 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -142,6 +142,7 @@ namespace Tensorflow.Keras.Engine int verbose = 1, List callbacks = null, IDatasetV2 validation_data = null, + int validation_step = 10, // 间隔多少次会进行一次验证 bool shuffle = true, int initial_epoch = 0, int max_queue_size = 10, @@ -164,11 +165,11 @@ namespace Tensorflow.Keras.Engine }); - return FitInternal(data_handler, epochs, verbose, callbacks, validation_data: validation_data, + return FitInternal(data_handler, epochs, validation_step, verbose, callbacks, validation_data: validation_data, train_step_func: train_step_function); } - History FitInternal(DataHandler data_handler, int epochs, int verbose, List callbackList, IDatasetV2 validation_data, + History FitInternal(DataHandler data_handler, int epochs, int validation_step, int verbose, List callbackList, IDatasetV2 validation_data, Func> train_step_func) { stop_training = false; @@ -207,6 +208,9 @@ namespace Tensorflow.Keras.Engine if (validation_data != null) { + if (validation_step > 0 && epoch ==0 || (epoch) % validation_step != 0) + continue; + var val_logs = evaluate(validation_data); foreach(var log in val_logs) { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index f86de8a8..0ca62c39 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -393,7 +393,7 @@ namespace Tensorflow.Keras.Layers.Rnn } } - public override Tensors Apply(Tensors inputs, Tensors initial_states = null, bool training = false, IOptionalArgs? optional_args = null) + public override Tensors Apply(Tensors inputs, Tensors initial_states = null, bool? training = false, IOptionalArgs? optional_args = null) { RnnOptionalArgs? rnn_optional_args = optional_args as RnnOptionalArgs; if (optional_args is not null && rnn_optional_args is null) From 7165304ff8b609f40bcbb24c0912a370d2c811ae Mon Sep 17 00:00:00 2001 From: dogvane Date: Mon, 10 Jul 2023 00:53:55 +0800 Subject: [PATCH 47/77] add a fucntion to cover a folder to a image classes dataset. --- ...processing.image_dataset_from_directory.cs | 1 + ...eprocessing.paths_and_labels_to_dataset.cs | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs index 4acae426..f42d12cd 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs @@ -58,6 +58,7 @@ namespace Tensorflow.Keras if (shuffle) dataset = dataset.shuffle(batch_size * 8, seed: seed); dataset = dataset.batch(batch_size); + dataset.class_names = class_name_list; return dataset; } diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs index b4d58387..eaa762d8 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs @@ -6,6 +6,31 @@ namespace Tensorflow.Keras { public partial class Preprocessing { + + /// + /// 图片路径转为数据处理用的dataset + /// + /// + /// + /// + /// + /// 用于调整大小的插值方法。支持`bilinear`、`nearest`、`bicubic`、`area`、`lanczos3`、`lanczos5`、`gaussian`、`mitchellcubic`。 + /// 默认为`'bilinear'`。 + /// + /// + public IDatasetV2 paths_to_dataset(string[] image_paths, + Shape image_size, + int num_channels = 3, + int num_classes = 6, + string interpolation = "bilinear") + { + var path_ds = tf.data.Dataset.from_tensor_slices(image_paths); + var img_ds = path_ds.map(x => path_to_image(x, image_size, num_channels, interpolation)); + var label_ds = dataset_utils.labels_to_dataset(new int[num_classes] , "", num_classes); + + return img_ds; + } + public IDatasetV2 paths_and_labels_to_dataset(string[] image_paths, Shape image_size, int num_channels, From b2fe5ca080dba6bd6473e386696d7c84191dc7ba Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 9 Jul 2023 15:35:23 -0500 Subject: [PATCH 48/77] Fix LSTM crash in release mode. --- src/TensorFlowNET.Core/Gradients/nn_grad.cs | 4 +- .../Operations/Operation.cs | 9 +-- .../Tensorflow.Binding.csproj | 72 +++++++++++++++-- .../Tensors/Tensor.Index.cs | 3 +- src/TensorFlowNET.Keras/Activations.cs | 1 - .../Callbacks/Earlystopping.cs | 3 - .../DataAdapters/TensorLikeDataAdapter.cs | 2 +- src/TensorFlowNET.Keras/Engine/Layer.Apply.cs | 2 +- .../Layer.FunctionalConstructionCall.cs | 10 --- .../Engine/Model.Evaluate.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.Train.cs | 4 +- src/TensorFlowNET.Keras/KerasInterface.cs | 4 +- .../Layers/Attention/BaseDenseAttention.cs | 10 +-- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 3 - .../Layers/Reshaping/Cropping1D.cs | 1 - src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 5 +- ...tUtils.get_training_or_validation_split.cs | 2 +- .../Saving/KerasObjectLoader.cs | 1 - .../Tensorflow.Keras.csproj | 79 +++++++++++++++++-- 19 files changed, 160 insertions(+), 57 deletions(-) diff --git a/src/TensorFlowNET.Core/Gradients/nn_grad.cs b/src/TensorFlowNET.Core/Gradients/nn_grad.cs index 3a6efd54..a43a91b9 100644 --- a/src/TensorFlowNET.Core/Gradients/nn_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/nn_grad.cs @@ -377,8 +377,8 @@ namespace Tensorflow.Gradients grad, op.get_attr_list("ksize"), op.get_attr_list("strides"), - op.get_attr("padding").ToString(), - op.get_attr("data_format").ToString()) + op.get_attr("padding"), + op.get_attr("data_format")) }; } diff --git a/src/TensorFlowNET.Core/Operations/Operation.cs b/src/TensorFlowNET.Core/Operations/Operation.cs index d31b26d4..e59c381c 100644 --- a/src/TensorFlowNET.Core/Operations/Operation.cs +++ b/src/TensorFlowNET.Core/Operations/Operation.cs @@ -206,12 +206,11 @@ namespace Tensorflow return result; } - internal unsafe int _get_attr_int(string name) + internal unsafe long _get_attr_int(string name) { - Status status = new(); - int result; - c_api.TF_OperationGetAttrInt(_handle, name, new IntPtr(&result), status); - status.Check(true); + long result; + c_api.TF_OperationGetAttrInt(_handle, name, new IntPtr(&result), tf.Status); + tf.Status.Check(true); return result; } diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 6a2dcff7..ca5aa47a 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -4,11 +4,11 @@ netstandard2.0;net6.0 Tensorflow.Binding Tensorflow - 2.10.0 - 0.110.1 + 2.11.0 + 0.110.2 10.0 enable - Haiping Chen, Meinrad Recheis, Eli Belash + Haiping Chen, Eli Belash, Yaohui Liu, Meinrad Recheis SciSharp STACK False Apache 2.0, Haiping Chen $([System.DateTime]::UtcNow.ToString(yyyy)) @@ -23,7 +23,8 @@ https://tensorflownet.readthedocs.io 0.110.1.0 tf.net 0.110.x and above are based on tensorflow native 2.11.0 - * RNN, LSTM works. + * Support RNN, LSTM model. + * Support Transformer model. tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -42,12 +43,11 @@ https://tensorflownet.readthedocs.io tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. - 0.110.1.0 + 0.110.2.0 LICENSE true packages true - Open.snk AnyCPU;x64 TensorFlow.NET Debug;Release;GPU @@ -88,6 +88,66 @@ https://tensorflownet.readthedocs.io + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + + + 1 + $(NoWarn),1570,1573,1591,1712,8603,8604,8625,CS0612 + + diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs index c8f47825..217712fe 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs @@ -180,8 +180,7 @@ namespace Tensorflow array_ops.stack(end.ToArray()), array_ops.stack(strides.ToArray())); - return gen_array_ops.strided_slice( - this, + return array_ops.strided_slice(this, packed_begin, packed_end, packed_strides, diff --git a/src/TensorFlowNET.Keras/Activations.cs b/src/TensorFlowNET.Keras/Activations.cs index d6d8e391..ce5b4eb1 100644 --- a/src/TensorFlowNET.Keras/Activations.cs +++ b/src/TensorFlowNET.Keras/Activations.cs @@ -44,7 +44,6 @@ namespace Tensorflow.Keras /// /// Register the name-activation mapping in this static class. /// - /// /// private static void RegisterActivation(Activation activation) { diff --git a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs index b3b78423..36993b63 100644 --- a/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs +++ b/src/TensorFlowNET.Keras/Callbacks/Earlystopping.cs @@ -5,9 +5,6 @@ namespace Tensorflow.Keras.Callbacks; /// /// Stop training when a monitored metric has stopped improving. /// -/// -/// - public class EarlyStopping: ICallback { int _paitence; diff --git a/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs b/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs index b93c6aed..16e646a3 100644 --- a/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs +++ b/src/TensorFlowNET.Keras/Engine/DataAdapters/TensorLikeDataAdapter.cs @@ -52,7 +52,7 @@ namespace Tensorflow.Keras.Engine.DataAdapters /// /// Convert a Tensor of indices into a dataset of batched indices. /// - /// + /// /// IDatasetV2 slice_batch_indices(Tensor indices) { diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs index 8a66948b..a3831bff 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Apply.cs @@ -10,7 +10,7 @@ namespace Tensorflow.Keras.Engine /// Wraps `call`, applying pre- and post-processing steps. /// /// - /// + /// /// /// public virtual Tensors Apply(Tensors inputs, Tensors states = null, bool? training = false, IOptionalArgs? optional_args = null) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.FunctionalConstructionCall.cs b/src/TensorFlowNET.Keras/Engine/Layer.FunctionalConstructionCall.cs index 1d96e581..e4023c3f 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.FunctionalConstructionCall.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.FunctionalConstructionCall.cs @@ -1,7 +1,5 @@ using System; using Tensorflow.Keras.Utils; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; namespace Tensorflow.Keras.Engine { @@ -9,14 +7,6 @@ namespace Tensorflow.Keras.Engine { Tensors FunctionalConstructionCall(Tensors inputs) { - bool mask_arg_passed_by_framework = false; - bool training_arg_passed_by_framework = false; - Tensor training_value = null; - if (training_value == null) - { - training_arg_passed_by_framework = true; - } - if (base_layer_utils.needs_keras_history(inputs)) base_layer_utils.create_keras_history(inputs); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index c4761f87..a74a77f1 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -15,7 +15,7 @@ namespace Tensorflow.Keras.Engine public partial class Model { /// - /// Returns the loss value & metrics values for the model in test mode. + /// Returns the loss value and metrics values for the model in test mode. /// /// /// diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index 48c16e18..ad3c70d2 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -29,7 +29,9 @@ namespace Tensorflow.Keras.Engine /// /// The logic for one training step. /// - /// + /// + /// + /// /// Dictionary train_step(DataHandler data_handler, Tensors x, Tensors y) { diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index 159564aa..6bc38109 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -72,8 +72,8 @@ namespace Tensorflow.Keras /// /// `Model` groups layers into an object with training and inference features. /// - /// - /// + /// + /// /// public IModel Model(Tensors inputs, Tensors outputs, string name = null) => new Functional(inputs, outputs, name: name); diff --git a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs index 19b29272..970a938d 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs @@ -1,24 +1,18 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.ArgsDefinition; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; using System; using System.Collections.Generic; using System.Linq; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; -/// -/// Base class for attention layers that can be used in sequence DNN/CNN models. -///This file follows the terminology of https://arxiv.org/abs/1706.03762 Figure 2. -///Attention is formed by three tensors: Query, Key and Value. -/// - namespace Tensorflow.Keras.Layers { /// /// Base Attention class for Dense networks. + /// This file follows the terminology of https://arxiv.org/abs/1706.03762 Figure 2. + /// Attention is formed by three tensors: Query, Key and Value. /// This class is suitable for Dense or CNN networks, and not for RNN networks. /// Implementations of attention mechanisms should inherit from this class, and /// reuse the `apply_attention_scores()` method. diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index d2080337..213b53a8 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -183,9 +183,6 @@ namespace Tensorflow.Keras.Layers /// Boolean, whether the layer uses a bias vector. /// The name of the initializer for the kernel weights matrix (see keras.initializers). /// The name of the initializer for the bias vector (see keras.initializers). - /// The name of the regularizer function applied to the kernel weights matrix (see keras.regularizers). - /// The name of the regularizer function applied to the bias vector (see keras.regularizers). - /// The name of the regularizer function applied to the output of the layer (its "activation") (see keras.regularizers). /// A tensor of rank 4+ representing activation(conv2d(inputs, kernel) + bias). public ILayer Conv2D(int filters, Shape kernel_size = null, diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs index 31285438..7d5385e6 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs @@ -2,7 +2,6 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; -using Tensorflow.Common.Types; namespace Tensorflow.Keras.Layers.Reshaping { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0ca62c39..6075547b 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -242,10 +242,9 @@ namespace Tensorflow.Keras.Layers.Rnn /// /// /// - /// Binary tensor of shape [batch_size, timesteps] indicating whether a given timestep should be masked - /// /// List of initial state tensors to be passed to the first call of the cell - /// List of constant tensors to be passed to the cell at each timestep + /// + /// /// /// /// diff --git a/src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.get_training_or_validation_split.cs b/src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.get_training_or_validation_split.cs index 2f3d8f52..18ca404e 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.get_training_or_validation_split.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.get_training_or_validation_split.cs @@ -6,7 +6,7 @@ namespace Tensorflow.Keras.Preprocessings public partial class DatasetUtils { /// - /// Potentially restict samples & labels to a training or validation split. + /// Potentially restict samples and labels to a training or validation split. /// /// /// diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index 1e869d66..fd1453d3 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -693,7 +693,6 @@ namespace Tensorflow.Keras.Saving /// Infers input shape of layer from SavedModel functions. /// /// - /// /// private TensorSpec _infer_inputs(int layer_node_id) { diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index ab667519..c7fa7711 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.1 + 0.11.2 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2023 @@ -26,7 +26,8 @@ * Add Subtract layer * Text preprocessing * Preprocessing.timeseries_dataset_from_array -* Fixed memory leak for YOLOv3 model. +* Fixed memory leak for YOLOv3 model. +* Support RNN and LSTM models Keras for .NET Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear & actionable error messages. @@ -36,10 +37,10 @@ Keras is an API designed for human beings, not machines. Keras follows best prac true packages Git - true + False Open.snk - 0.11.1.0 - 0.11.1.0 + 0.11.2.0 + 0.11.2.0 LICENSE Debug;Release;GPU @@ -70,6 +71,74 @@ Keras is an API designed for human beings, not machines. Keras follows best prac + + True + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + True + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + True + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + True + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + False + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + False + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + False + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + False + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + + + 1 + $(NoWarn),1573,1591,1712,8602,8603,8625,CS0612 + + From f56811d080f8891a396831c39073b687e1733302 Mon Sep 17 00:00:00 2001 From: dogvane Date: Tue, 11 Jul 2023 02:31:32 +0800 Subject: [PATCH 49/77] fix flip_left_right run bug --- src/TensorFlowNET.Core/Operations/image_ops_impl.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 126df9e4..0ced407a 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -208,7 +208,7 @@ namespace Tensorflow } public static Tensor flip_left_right(Tensor image) - => _flip(image, 1, "flip_left_right"); + => _flip(image, 0, "flip_left_right"); public static Tensor flip_up_down(Tensor image) => _flip(image, 1, "flip_up_down"); @@ -226,7 +226,7 @@ namespace Tensorflow } else if (shape.ndim == 4) { - return gen_array_ops.reverse(image, ops.convert_to_tensor(new[] { flip_index + 1 })); + return gen_array_ops.reverse_v2(image, ops.convert_to_tensor(new[] { (flip_index + 1) % 2 })); } else { From 70f873eccef99e4ca6af39a8ac798cc36292ace2 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 10 Jul 2023 15:02:39 -0500 Subject: [PATCH 50/77] Initially adding KerasTensor. #1142 --- src/TensorFlowNET.Core/GlobalUsing.cs | 4 +- .../Keras/Layers/ILayersApi.cs | 3 +- src/TensorFlowNET.Core/Tensors/KerasTensor.cs | 40 +++++++++++++++++++ src/TensorFlowNET.Keras/BackendImpl.cs | 2 +- src/TensorFlowNET.Keras/GlobalUsing.cs | 3 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 2 +- 6 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 src/TensorFlowNET.Core/Tensors/KerasTensor.cs diff --git a/src/TensorFlowNET.Core/GlobalUsing.cs b/src/TensorFlowNET.Core/GlobalUsing.cs index 2fd5b437..209bc291 100644 --- a/src/TensorFlowNET.Core/GlobalUsing.cs +++ b/src/TensorFlowNET.Core/GlobalUsing.cs @@ -3,4 +3,6 @@ global using System.Collections.Generic; global using System.Text; global using System.Collections; global using System.Data; -global using System.Linq; \ No newline at end of file +global using System.Linq; +global using Tensorflow.Keras.Engine; +global using Tensorflow.Framework.Models; \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 9bc99701..b48cd553 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,5 +1,6 @@ using System; using Tensorflow.Framework.Models; +using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers.Rnn; using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; @@ -135,7 +136,7 @@ namespace Tensorflow.Keras.Layers public ILayer GlobalMaxPooling1D(string data_format = "channels_last"); public ILayer GlobalMaxPooling2D(string data_format = "channels_last"); - public Tensors Input(Shape shape = null, + public KerasTensor Input(Shape shape = null, int batch_size = -1, string name = null, TF_DataType dtype = TF_DataType.DtInvalid, diff --git a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs new file mode 100644 index 00000000..1034dcc8 --- /dev/null +++ b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs @@ -0,0 +1,40 @@ +namespace Tensorflow.Keras.Engine; + +/// +/// A representation of a Keras in/output during Functional API construction. +/// +public class KerasTensor +{ + private Tensor _tensor; + public void SetTensor(Tensors tensor) + => _tensor = tensor; + + private TensorSpec _type_spec; + private string _name; + + public KerasTensor(TensorSpec type_spec, string name = null) + { + _type_spec = type_spec; + _name = name; + } + + public static KerasTensor from_tensor(Tensor tensor) + { + var type_spec = tensor.ToTensorSpec(); + var kt = new KerasTensor(type_spec, name: tensor.name); + kt.SetTensor(tensor); + return kt; + } + + public static implicit operator Tensors(KerasTensor kt) + => kt._tensor; + + public static implicit operator Tensor(KerasTensor kt) + => kt._tensor; + + public static implicit operator KerasTensor(Tensor tensor) + => from_tensor(tensor); + + public static implicit operator KerasTensor(Tensors tensors) + => from_tensor(tensors.First()); +} diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 364800ae..574cf599 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -76,7 +76,7 @@ namespace Tensorflow.Keras _GRAPH_VARIABLES[graph.graph_key] = v; } - public Tensor placeholder(Shape shape = null, + public KerasTensor placeholder(Shape shape = null, int ndim = -1, TF_DataType dtype = TF_DataType.DtInvalid, bool sparse = false, diff --git a/src/TensorFlowNET.Keras/GlobalUsing.cs b/src/TensorFlowNET.Keras/GlobalUsing.cs index bc0798ed..85cd9194 100644 --- a/src/TensorFlowNET.Keras/GlobalUsing.cs +++ b/src/TensorFlowNET.Keras/GlobalUsing.cs @@ -4,4 +4,5 @@ global using System.Text; global using System.Linq; global using static Tensorflow.Binding; global using static Tensorflow.KerasApi; -global using Tensorflow.NumPy; \ No newline at end of file +global using Tensorflow.NumPy; +global using Tensorflow.Keras.Engine; \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 213b53a8..5968461d 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -466,7 +466,7 @@ namespace Tensorflow.Keras.Layers /// In this case, values of 'None' in the 'shape' argument represent ragged dimensions. For more information about RaggedTensors, see this guide. /// /// A tensor. - public Tensors Input(Shape shape = null, + public KerasTensor Input(Shape shape = null, int batch_size = -1, string name = null, TF_DataType dtype = TF_DataType.DtInvalid, From ed1a8d2edfbad3e47efa48af5e1dbb4c22a20f2e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 10 Jul 2023 23:00:26 -0500 Subject: [PATCH 51/77] Add shape and dtype to KerasTensor --- .../Operations/array_ops.cs | 49 ++++++++++++------- src/TensorFlowNET.Core/Tensors/KerasTensor.cs | 27 +++++++--- .../Tensors/Tensor.Index.cs | 2 +- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 5237ec44..02bf0e86 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -603,7 +603,17 @@ namespace Tensorflow } } - return gen_array_ops.shape(input, name: name, out_type: out_type); + return tf.Context.ExecuteOp("Shape", name, new ExecuteOpArgs(input) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + out_type = op.get_attr("out_type") + } + }.SetAttributes(new + { + out_type + })).First(); }); } @@ -703,23 +713,26 @@ namespace Tensorflow int new_axis_mask = 0, int shrink_axis_mask = 0, string name = null) - { - var op = gen_array_ops.strided_slice( - input: input_, - begin: begin, - end: end, - strides: strides, - begin_mask: begin_mask, - end_mask: end_mask, - ellipsis_mask: ellipsis_mask, - new_axis_mask: new_axis_mask, - shrink_axis_mask: shrink_axis_mask, - name: name); - - string parent_name = name; - - return op; - } + => tf.Context.ExecuteOp("StridedSlice", name, new ExecuteOpArgs(input_, begin, end, strides) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + Index = op.get_attr("Index"), + begin_mask = op.get_attr("begin_mask"), + end_mask = op.get_attr("end_mask"), + ellipsis_mask = op.get_attr("ellipsis_mask"), + new_axis_mask = op.get_attr("new_axis_mask"), + shrink_axis_mask = op.get_attr("shrink_axis_mask") + } + }.SetAttributes(new + { + begin_mask, + end_mask, + ellipsis_mask, + new_axis_mask, + shrink_axis_mask + })); /// /// Returns the gradient of `StridedSlice`. diff --git a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs index 1034dcc8..3204b4ac 100644 --- a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs +++ b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs @@ -5,12 +5,17 @@ /// public class KerasTensor { - private Tensor _tensor; - public void SetTensor(Tensors tensor) - => _tensor = tensor; + private Tensors _inferred_value; + public Tensors inferred_value + { + get => _inferred_value; + set => _inferred_value = value; + } - private TensorSpec _type_spec; private string _name; + private TensorSpec _type_spec; + public Shape shape => _type_spec.shape; + public TF_DataType dtype => _type_spec.dtype; public KerasTensor(TensorSpec type_spec, string name = null) { @@ -22,15 +27,23 @@ public class KerasTensor { var type_spec = tensor.ToTensorSpec(); var kt = new KerasTensor(type_spec, name: tensor.name); - kt.SetTensor(tensor); + kt.inferred_value = tensor; return kt; } + public override string ToString() + => _inferred_value.Length switch + { + > 1 => "[" + string.Join(", ", _inferred_value.Select(x => $"")) + "]", + 1 => $"", + _ => _inferred_value.ToString(), + }; + public static implicit operator Tensors(KerasTensor kt) - => kt._tensor; + => kt._inferred_value; public static implicit operator Tensor(KerasTensor kt) - => kt._tensor; + => kt._inferred_value; public static implicit operator KerasTensor(Tensor tensor) => from_tensor(tensor); diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs index 217712fe..51062cf3 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Index.cs @@ -42,7 +42,7 @@ namespace Tensorflow array_ops.stack(args.End), array_ops.stack(args.Strides)); - return gen_array_ops.strided_slice( + return array_ops.strided_slice( this, packed_begin, packed_end, From b27ccca84fe68394e4ffbf4babd16d0d2e05674e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 11 Jul 2023 22:40:30 +0800 Subject: [PATCH 52/77] fix:fix the bug of load LSTM model --- .../Keras/ArgsDefinition/Rnn/GRUCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 12 ++++++-- .../ArgsDefinition/Rnn/RnnOptionalArgs.cs | 2 +- .../Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs | 2 +- .../ArgsDefinition/Rnn/SimpleRNNCellArgs.cs | 2 +- .../ArgsDefinition/Rnn/StackedRNNCellsArgs.cs | 4 +-- .../Keras/Layers/ILayersApi.cs | 2 +- .../Keras/Layers/Rnn/IRnnCell.cs | 2 +- .../Keras/Layers/Rnn/IStackedRnnCells.cs | 2 +- .../Operations/NnOps/RNNCell.cs | 3 +- src/TensorFlowNET.Core/ops.cs | 4 ++- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 3 +- .../Layers/Rnn/DropoutRNNCellMixin.cs | 2 +- src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs | 3 +- src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 4 +-- .../Layers/Rnn/LSTMCell.cs | 4 +-- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 4 +-- src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs | 2 +- .../Layers/Rnn/SimpleRNN.cs | 4 +-- .../Layers/Rnn/SimpleRNNCell.cs | 4 +-- .../Layers/Rnn/StackedRNNCells.cs | 4 +-- .../Saving/KerasObjectLoader.cs | 1 - .../SavedModel/serialized_attributes.cs | 2 +- src/TensorFlowNET.Keras/Utils/RnnUtils.cs | 2 +- .../Layers/Rnn.Test.cs | 2 +- .../Model/ModelLoadTest.cs | 29 ++++++++++++++++++- 28 files changed, 71 insertions(+), 40 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs index 75d5d021..624756af 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUCellArgs.cs @@ -3,7 +3,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class GRUCellArgs : AutoSerializeLayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index db76fda0..d816b0ff 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class LSTMArgs : RNNArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs index 786236e4..f4503231 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -1,7 +1,7 @@ using Newtonsoft.Json; using static Tensorflow.Binding; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { // TODO: complete the implementation public class LSTMCellArgs : AutoSerializeLayerArgs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index 2d7fb001..b84d30d3 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -1,8 +1,8 @@ using Newtonsoft.Json; using System.Collections.Generic; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { // TODO(Rinne): add regularizers. public class RNNArgs : AutoSerializeLayerArgs @@ -23,16 +23,22 @@ namespace Tensorflow.Keras.ArgsDefinition.Rnn public int? InputDim { get; set; } public int? InputLength { get; set; } // TODO: Add `num_constants` and `zero_output_for_mask`. - + [JsonProperty("units")] public int Units { get; set; } + [JsonProperty("activation")] public Activation Activation { get; set; } + [JsonProperty("recurrent_activation")] public Activation RecurrentActivation { get; set; } + [JsonProperty("use_bias")] public bool UseBias { get; set; } = true; public IInitializer KernelInitializer { get; set; } public IInitializer RecurrentInitializer { get; set; } public IInitializer BiasInitializer { get; set; } + [JsonProperty("dropout")] public float Dropout { get; set; } = .0f; + [JsonProperty("zero_output_for_mask")] public bool ZeroOutputForMask { get; set; } = false; + [JsonProperty("recurrent_dropout")] public float RecurrentDropout { get; set; } = .0f; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs index 64b500bb..a6520589 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RnnOptionalArgs.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Common.Types; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class RnnOptionalArgs: IOptionalArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs index fcfd694d..e45ef79d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class SimpleRNNArgs : RNNArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs index d21d6190..b84ea21b 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNCellArgs.cs @@ -1,6 +1,6 @@ using Newtonsoft.Json; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class SimpleRNNCellArgs: AutoSerializeLayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index 50a6127d..2600f14e 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; -namespace Tensorflow.Keras.ArgsDefinition.Rnn +namespace Tensorflow.Keras.ArgsDefinition { public class StackedRNNCellsArgs : LayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index b48cd553..1670f9d1 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,7 +1,7 @@ using System; using Tensorflow.Framework.Models; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs index 8d6fbc97..43df75b1 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IRnnCell.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Common.Types; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public interface IRnnCell: ILayer { diff --git a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs index e73244a5..8cf6150d 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/Rnn/IStackedRnnCells.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public interface IStackedRnnCells : IRnnCell { diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 4e99731f..9905d39c 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -19,9 +19,8 @@ using System.Collections.Generic; using Tensorflow.Common.Types; using Tensorflow.Keras; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Operations; diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 2dc46329..c624c990 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -571,7 +571,9 @@ namespace Tensorflow if (tf.Context.executing_eagerly()) return true; else - throw new NotImplementedException(""); + // TODO(Wanglongzhi2001), implement the false case + return true; + //throw new NotImplementedException(""); } public static bool inside_function() diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 5968461d..cb85bbba 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -2,9 +2,8 @@ using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Core; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs index 75feb8ea..27c13f34 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/DropoutRNNCellMixin.cs @@ -6,7 +6,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public abstract class DropoutRNNCellMixin: Layer, IRnnCell { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs index 02fe54f4..2b9c01e3 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/GRUCell.cs @@ -3,12 +3,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.Text; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; using Tensorflow.Keras.Saving; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for the GRU layer. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index 025465fd..b5d58324 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -1,10 +1,10 @@ using System.Linq; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Long Short-Term Memory layer - Hochreiter 1997. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 284a2b77..e4fc6dd2 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -3,12 +3,12 @@ using Serilog.Core; using System.Diagnostics; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for the LSTM layer. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 6075547b..0e81d20e 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Reflection; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Util; @@ -14,7 +13,7 @@ using Tensorflow.Common.Types; using System.Runtime.CompilerServices; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Base class for recurrent layers. @@ -185,6 +184,7 @@ namespace Tensorflow.Keras.Layers.Rnn public override void build(KerasShapesWrapper input_shape) { + _buildInputShape = input_shape; input_shape = new KerasShapesWrapper(input_shape.Shapes[0]); InputSpec get_input_spec(Shape shape) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs index 018b1778..1419da4b 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RnnBase.cs @@ -4,7 +4,7 @@ using System.Text; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public abstract class RnnBase: Layer { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index a22f31c7..9c199eb4 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -1,11 +1,11 @@ using System.Data; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; using Tensorflow.Operations.Activation; using static HDF.PInvoke.H5Z; using static Tensorflow.ApiDef.Types; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public class SimpleRNN : RNN { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index c77f7779..e74b5692 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; using System.Text; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Common.Types; @@ -9,7 +9,7 @@ using Tensorflow.Common.Extensions; using Tensorflow.Keras.Utils; using Tensorflow.Graphs; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { /// /// Cell class for SimpleRNN. diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 8799bfb2..ece2bc5b 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -3,12 +3,12 @@ using System.ComponentModel; using System.Linq; using Tensorflow.Common.Extensions; using Tensorflow.Common.Types; -using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; using Tensorflow.Keras.Utils; -namespace Tensorflow.Keras.Layers.Rnn +namespace Tensorflow.Keras.Layers { public class StackedRNNCells : Layer, IRnnCell { diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index fd1453d3..0bd816cc 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -13,7 +13,6 @@ using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; -using Tensorflow.Keras.Layers.Rnn; using Tensorflow.Keras.Losses; using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Saving.SavedModel; diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs index 0ec5d1a8..325d3327 100644 --- a/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Metrics; using Tensorflow.Train; diff --git a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs index e8700c1f..1e9f6d84 100644 --- a/src/TensorFlowNET.Keras/Utils/RnnUtils.cs +++ b/src/TensorFlowNET.Keras/Utils/RnnUtils.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Text; using Tensorflow.Common.Types; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Common.Extensions; namespace Tensorflow.Keras.Utils diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index becdbcd6..5f7bd574 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -6,7 +6,7 @@ using System.Text; using System.Threading.Tasks; using Tensorflow.Common.Types; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.NumPy; using Tensorflow.Train; diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 10db2bd1..382941d9 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -1,5 +1,7 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Linq; +using Tensorflow.Keras.Engine; using Tensorflow.Keras.Optimizers; using Tensorflow.Keras.UnitTest.Helpers; using Tensorflow.NumPy; @@ -79,6 +81,31 @@ public class ModelLoadTest model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); } + [TestMethod] + public void LSTMLoad() + { + var inputs = np.random.randn(10, 5, 3); + var outputs = np.random.randn(10, 1); + var model = keras.Sequential(); + model.add(keras.Input(shape: (5, 3))); + var lstm = keras.layers.LSTM(32); + + model.add(lstm); + + model.add(keras.layers.Dense(1, keras.activations.Sigmoid)); + + model.compile(optimizer: keras.optimizers.Adam(), + loss: keras.losses.MeanSquaredError(), + new[] { "accuracy" }); + + var result = model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 3, workers: 16, use_multiprocessing: true); + + model.save("LSTM_Random"); + + var model_loaded = keras.models.load_model("LSTM_Random"); + model_loaded.summary(); + } + [Ignore] [TestMethod] public void VGG19() From 9c949e336f6c9fedd6a6ae5eace581084d16a8b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 11 Jul 2023 23:44:42 +0800 Subject: [PATCH 53/77] refactor: refactor LSTMLoad test --- .../lstm_from_sequential/fingerprint.pb | 1 + .../lstm_from_sequential/keras_metadata.pb | 7 +++++ .../lstm_from_sequential/saved_model.pb | Bin 0 -> 755111 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 61038 bytes .../variables/variables.index | Bin 0 -> 1373 bytes .../Model/ModelLoadTest.cs | 26 ++++-------------- .../Tensorflow.Keras.UnitTest.csproj | 16 +++++++++++ 7 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.index diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb new file mode 100644 index 00000000..f6ea8da2 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb @@ -0,0 +1 @@ +沦Ʉ%̟땐͉ Σ(Ћ܇}2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb new file mode 100644 index 00000000..5fe8f1a6 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/keras_metadata.pb @@ -0,0 +1,7 @@ + +&root"_tf_keras_sequential*&{"name": "sequential", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "shared_object_id": 9, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 5, 3]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 5, 3]}, "float32", "input_1"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 5, 3]}, "float32", "input_1"]}, "keras_version": "2.12.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 5, 3]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "shared_object_id": 0}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 5}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 6}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 7}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 8}]}}, "training_config": {"loss": "binary_crossentropy", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "accuracy", "dtype": "float32", "fn": "binary_accuracy"}, "shared_object_id": 11}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Custom>Adam", "config": {"name": "Adam", "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "jit_compile": false, "is_legacy_optimizer": false, "learning_rate": 0.0010000000474974513, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}}}}2 + root.layer_with_weights-0"_tf_keras_rnn_layer* {"name": "lstm", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "return_sequences": false, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 5, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, null, 3]}, "ndim": 3, "max_ndim": null, "min_ndim": null, "axes": {}}, "shared_object_id": 12}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 5, 3]}}2 +root.layer_with_weights-1"_tf_keras_layer*{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 6}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 7}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 8, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 32}}, "shared_object_id": 13}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 32]}}2 +root.layer_with_weights-0.cell"_tf_keras_layer*{"name": "lstm_cell", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "LSTMCell", "config": {"name": "lstm_cell", "trainable": true, "dtype": "float32", "units": 32, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}, "shared_object_id": 2}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 3}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}, "shared_object_id": 4, "build_input_shape": {"class_name": "__tuple__", "items": [null, 3]}}2 +Rroot.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 14}2 +Sroot.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "accuracy", "dtype": "float32", "config": {"name": "accuracy", "dtype": "float32", "fn": "binary_accuracy"}, "shared_object_id": 11}2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..6fb7c3f0e8e4a35afa38cada60f78a6097b84501 GIT binary patch literal 755111 zcmeFaeQaDwb}z=;&5ve@qNpoUqDqve#aYcq&v3|Yez$kmGt%zN&hE@eD`{rdv%7QE zVmGB8kxg!QQ#0BHvTMI*@8bEz2%N-#9XrO3jkV+Vhn6jQ4;b zuwgvdj^WrbV8s6f&Z&=k>(sqf_oI19+W}{>6nSsmsye?qb?VfqQ|FW%_q#v-68R6* z{qHFG3>kT)*?gzwjuPjidFL+s_X+$<;oq;rzw?&~JRI&n*>BZdGF6+LJm(aK&z`?9 zI#L`jx!-b}|HN^~`()~sUa$S2^G>7NZrtB#-Q0I4NU7WE?HzQtTC>ycBpG?!*g0s; zk2G0b^QVdUMN~ac$t1jPHyizy*L&F5Z@I(w_x5&Z;klvjK>gox$k_FEqX%7eiC+`+ zYn%+o6)S%17;gQkO_^FB0lV_&Ry64Vcm|q~Lpt**( zz1Q7s^xe^3zuWFSprh{aTc3OPMtzb@h~MDnjnBPHHps}0y-u%B&jbxgQqIT@xugA7 zr?=M)(MCQ&&VHudW2oHN+kbMibGy}O>gX^M-zUSbws&@1mkc-CyS;hR?lfB;c?>K} zvQFoCKD`Y0K$6^i5JB~p65+3NJ${U@xxc1J*EtB)Lz(Kj3YHxG6g zM>GIb_V0a$^aL5(@9s4Zwm8%m$f4icHof7)KSy z8Bdj5@?#Eh-)waswBD)Rc-Y!{w7=Kx^n30Fa&9a5)7$U1w%Z@Idb4BhdEz!(y?(pX z=(qPe^3Ckn{3UXx+1hCVJMHZ>y=J@H3wq$pKSjWYwzYhd6Nv^YHU4%88FIc0OQt*^Og?$1aaPSi$Lmg z&XO~ahV={?ZhX}4@prT^J4F6lhn%_9=)yPxy{+aAAjCZdg}M)Vwc_1&heA7IBGiWO z!Y}jZ2z%_3op!JP9Al^FpL6D?34a-S3hi#Y!`la)EhsR$wb$8hKL{p9!5xBWIw76{ z(0lzo?cF7Ey7f_O>!82a6PP*p3<{qOqA=@M!6vC&Ce3~n-1oZ!y4QnqjwsQ2W-vF+S(67YryndefKmOhkaxko{%x@-tO-lSZKO#??r`agCB!p6L(P|fpZM7R3+HUakxyHta$> zXjP%-1A<|uj`QYONthGr57T5N|HdH~?(|^|+dkO&CpSldU4I!k_GcV2$!P$I*n9UO zU>z(MHOx8i$NX7_u`XC9$S`9(DMjF~`+LnN@~<*EEhLjz;(P8XE|d5)d!81^5G-Qn z_8VOgX{{Zv-EVcdP!EJ30SPe8`qtjM-Fnda=+%Rr9ay8b9x{gE3mVK&nAM_@SpxkV z+i&!to{lf6$p7Pzi94{;1M}TslR6-dO%D(t_gfFzo%xa1+2RXaS!{Kh{$EpMjOhrl z+{h=kX*7SH$O?qsG)6{hMNpT3Cke0L+imn71tJ|JbP1XXJ)8w`y0g>X@3ni;H=Wjp z9$OYdZ!R&J1zOA_!%OlOA&l_OgU`c~3Ibx3BL+f^Ek+8pv3Y$hx3Hy5JWo+G12*Te6CGN5`0Qv>u2=mH;E&uh8{u+-3g+vRSP-^B0MDXILhF$|0vo?k+md2M*<0+uB+iLQq9J6dlF`tL&&hc=?Kl4wMGh8i%Pn)gB zY{hpcgoNVrhKneWlJg^+5h|m#m#VdwmTH$Ci(eMHTh)bXZQ;RE^Zx4QdUL6{zPjPK z|NEc62+IA)*4{y$HuzqFVAUxyb8Z zWcm^Rfs~|>(nQ5Ksaj_p@9*^+pkkttNcCw_$9c7^IIkH36L5qaZ+5%m|RbrXxTWPDjAk$=O$$jorn^i?HqNwHDb% zeSoom-L>HV(fY7kC^?mReLT*&(Pl`GT#K}wio|GL6);VNW@(A|eZe!8_yq4B55c&Z zfMEPE1Ht&S^aTHiL(XY1KLSmwHHfS@=MOTemtc^~e4sy}5A=*{YK(k?JZtDfD%wr; zRjx#adxGFzX+s_$-~)Ip2b8bw7%zpHey+5fyiK)JFDZ(A&BLBcs`UFV?Z+>{+4A3LO;uCJ*8OxbMZ zsR);yCK^E+rv-pUM&`HyP3(XIG_hy~KojfA2<-BGrjH<*KTQM=l5Lq#{Or zn}(2a@=bE?sPvbp9&2ccd_X`iN0?;WG2+DoMqlKI#d^2b9?M;{CQ(fQo&-bGzJXQI)NO;VI<&{E7tXXUXvJ6m+7*?$$VcC!F4%Rwz0S@P_)YlKh+*o% z(bpIq;r=a^h0y8zWjL!CBgGrH-kt*g8B%-=ei^D2K0mbK{D?CILkg$;&)oh5nfIWw z?H263w!r%aym%V@gKo?Fu-n)Nv%R;vv9dzoB#(JBfHMs9Y52K{^oDVTQ_TEnuz4|y zc^w+7FQr*>$14BsdEas7cb~uhzdm0YbBC_~&(D*eDUjKc<4*51oE8sE$E;*ihaS^z8 zrrUaOumfekF8oP$_IkZm52_e-$3YG>dR}9{O(vkPCC}S}(*n;Ul~T!lhX2l71-dN& zuj8>-!3lh0$CDLZDvi4rf|=F>*8l}Lv_jnRz5RZBxBc~2mrRyUxg;2lUImPw?1cGr zX+kIGw2hoeJ2_``a;7wL&ZZ`3rsTS3L8I+7_jbJpEjWr{$AqL*I_C~EBM{Iq4K%PL zaj&=E+Tt{PkI^ur({SEK11x*CA(_=_xTw)UQ`107m)ubyny=PS6rtG|2*Y7HTdU}>ny`RbFQQHOl(L>SKU$0;yvKw zd3S8L1;BCc-BN`499m3 zfVk$40a_rSm=Jiz|AS5SXZ=5F@W*rhAIktW`|f%F0sF4v|G~a{!T*DO_oDv?`;HCM z4}%{>X~88(K%6PPC3+YkB=*mp1cf3WXf@&91oUHAX+ne9)QNW7to#Lw77;%Dt5@l{t_ zrC!rS;^$I}#F^6TCb2$Q`g|bFVTA%lx*SN(X%LZ9;cX+x)2EO;W&Gy~@yTQ=h|el# zS<^FC_CIBk@(XeN`3B<8H!**{mBgPnqx^a3C268G=o<7)iKgbyGr&N%dVuBwY-5nO!fmAiLuzkdDgyPv(Y zsMo?r>r3wKNbN{5rYpv}*`k_M^V{ytNX-^){+U2+TU<)2)^mR-Qf=aH05woeYT%Ch zW~2e*ZUR+WORChLERIk>3V5>I7SHbZ&sfawTxXMvmcHs1s!N<5OvEeW8(j+Y*+sN+CK3Oh zdpClpZl0vesA6pePO6&$#WXuWfJq@BooRMJGt=yVSf<$ll}fV%j95VchZO`=SV4e; z75u7SFbqE!i$Do>1V-VPaev3G-!ZRPja*#5A15w7Brfk^aoI=`m-nOM^3coGL}}3V z>6y~!Qj5ztti$g%U=`lNtMKVk6R)V_rIsuFp~!e?TRLCyS@1dd@`3&(c%`Zx<9Mm9 zzWl&<+nOvr0@ZN4<30M2`S0_Z9=MGN>x(lR%TWZaqM4w?VxrWtH}sevR-+SZu`pN~ zN|a<@b00-05+|V%ioyznrOD-_Ic{$N!Sh z=82i8u!O}D`u7{P6@? z*9v?0;pz@-0-D}~{R593=ZllCZ8%3_ue^=r+J?8Wx&-E6a%x{)rJZ%y^&a%Vt41Hy zqK}rNk5=H(GFg1kZ8Y0ptL}NNkM`kYhuuc^Y94zm?(KQv7M#2l3m-jCuBnfDUpr{E zzTU#+w+@<}PPfy6qA!qVqeZKh!i(euRG44AlZ{$FxJoXo58w!GujlQwAGJKxfi+T7 zMf~TYI{z<*H(YPtXYxthYln`nljYRkDcs=0O|m+`hkO)QNj3GSaB{NM==0A$MK;xE z0S$u2b_cYQKD&%{!0u6sBPe-!pFM?}uod!*`dpEMA#Zo!-t0pl?nUyPv8*zN{cc^B@QdG! zxqPM*MezrQRwU~NKs$K#lFAanuo|6+4k`x~UU z_BV)b?Qf9V+TW|QEc5_?$|AZ?nGsJ&(uoI)=Eval9~=<3(K|^mIr9zriq3XTMH|y1&e}C!4dqb8r#qE zW%@H}nO?lZV>pOV4m9h6u9~;sd@Ix^z^SIhVn7s2zvs|jamdqg9fG^lyh~&97dod( zv)>qNR_o#}Y<-DV)LRR2L0Q$?)C2xaf?B_W)D<1KbUKY{-;P!>gw`)Qp>cz zY60<7^>I~~hC|IFA;Ap~`((&}`oBk95VWrvibV98#jfonOy zfvSE6aA9(Sf~udH0Q*gF8Btw@SXFPi!|bPt+n0jI=!@!`D7Q-rgVL;PZ@Gopg6bp0 zXYsan8oi!7N2ogsekJvnRh8^FF8R|-_8)i1ysALc_W3fc4}KOMU3EG=1Hg?wrFt)# zdLwoQJe%{U+K*q-tCM)yt^^4*lsOS%_zf&@i}DcS297Vih9^Y>5aMoqjS? zYhuvV*Y#{)!%(SjKb`GciM0BDT7A!^f4z6G3qG-MA}pfHKvXt059|VFBqEW)tu`9p z1p;HyErOa<`(AhaJ+=pa1Gd04eq*fqGaKS_5 zy4*UuQ{g9cfNeKb^FG+Zi+w_5tFHE`bg=C<&A!z}?5vwaL?5}v6Z9J`)n1H=1T9fP zF}dvs)wiaYEQL%|%xPEySxs=!fH$BDs=a!6v5XW=%Ut&&%=H#|11=G;djp}11V#Qh zs+HIt^Sk2LD8!6G%A+zGa1vkV#*bUd_~D+7Jfef!PGXjtBK>`BO$~J{_|sJ}bWleU zCR<%x*znC(pg+H)cq6Djg3X^z8ifXj-|}Gk;Zv$1q`GjhxeDsaJxEH6 zlInJ_T zG&+xb?Z~px#uu526*#LFs@eGUF;LBh>R@DnXnB_uN9hfK&Brahy0f>}eRHo#l{ocn zPI9=AX{gFgpJ=IA)N8yAU5=npwB5F$50E)LRFj;YH&{g57`es6Pl;%3OlvVKr}taWDSWF$6Wdxn#vV!UPqF9NU0Vw|KM6L$4;Fu|1NupRuOAEy@krzue znPkvTj+x}uB$AQTH=hk0l4RIh9W%*mNhB*p<7h@1ws^-(@_G`q0Flr&N?>;=q=u|*&#H4mrmypFW zNf|#HqI1!`vJqWPBwB4Nj)v%5)URwr*Aj_7gaTHkDcCy~4J;ec%ZWrELI+DlbS_F* zHlkM&iB43*;zW0fkr6*Sah1Qxr2jlqKbP?4`hsM1`a5#H;*R~@Xzv1f95KqFhs3XNI^ke`b0 zIU?@}2{Q8H8gb3yIH8d(W`#zr7A9yg8F@h?u8-sn4U;s~JfUQZS)oyDiU}G_MrLTr zW?3Vy#T+Lzvc;^>sFlYA4JIQmXvB4&~5f9DvLH5e4E*%9R5GWS8#D&|6AYwmd6M7vwJVYvBxLTWL7sn%JOtNXb zDVfBAiEJw~Cd!pFMDbl^*0rv1Wm=}a+a`-C7lJbTMD)MZE>mL1f zTTPMQueQM|rQG1KiV7eQwmsm4h_Mh0>#tj2mE8-_z_2174Hhz?p2$A~&Z9G=RFh4$dUiV9+Z6;VnC0#pk7FZGI zXCPo@_tF9@u}TN5jr455{%H~W*NV_Vat#$e8E#2Ee-MY+Oz{xy|x9u(HBSs^|?e z<`pAH$18%?utLk8)L5hxq~9+@=Z56AbeM%T0Xu!+rOjS4EUWD$!@`G`jBuGCNJkCD z5c9b#6>cIgCH)GaiqQm8a=Vkq4cDJ^$VGGY`u$erMBo$0*#q5fWr5Eq3KJFXY9LE| zp0VF--NthZ*OSucB4k9f+?90bL(Gwpb_TfCYHjJawfF$IsT=kiPb%($@1e&qU-a)g zMK4kD*Sdw7G*@6LeWCQbR@5A&ypL85L1y8d{MRR79EhwIZvih_ZOkUnV6#h$mQ|<&+A#T6vDl zu%!{A^&CJ(SO~offAm`)^<7Bc{^+wTH|*^LhzGjcs{Cz%oQ6Hp{y|?vT<*DlQXqd+ zh$e?EM>Fv9u$iWkk<7B?`jDv{3&)c_?v62dPniXb$}_GHUtNl(-WsW5!43-in{+>i z015bCwOV`+QUvPPDZLhh#yuG@#++0}RTG?D!*=$x9Hd^b|g{}MR ztBuY3Yptc`I)vMwW}okOTaZk*3z6+bPFxpa6G7_T%FASCd#}rbd~Y9g1Pk!o5YM{T zJkNh+i~kL+Lj;nE2YW%Na7ZKk+)!nT5XLi*UigE(E_+g+gS33J#1hg3-p<9oy%hTv z;tN`9E<^e|h(H+84oYNHjzzgNn^@vPVu|yKC1w&!OedB&msrA0C@}{~C+%o0Kr+kc zArV>lmaj1hbh*AzI+LY%xilHuyJG!n>9l*9#E9Q(^(tf2# zL)%qY1neUj3J4|^orTg+h8D=5kCR_7k~1GZY(pAuSPPrp>T+!j*mhPt53IPJQkMl86K}h38qE^Tnd*t*maYx`d2!^}h!*mj5pEZTP>lY#Q*p z$dL!XPm@UQWq4X5JeNJp;0d)efxix!Y=9*f*dmjG8DNnm+emguD(d%~uKM?!X0M-@ z_neCJ6As)yVE+~1|MW2#4;sM^9ugaRA$ZHXpr&vT&vGTAB0!d&`V~FcXlN}f(pzLw znE@6d|BX+aMW@yYi-rjtf?Eni)##$%DbT{kmR?ZrS&If*vgl#-HknctdevNr=#dFr zr-6(udJ&h%gOD$hGu&jcC=3^e9}I_n#~~xUHn>#Ha(B?#PzwS0RL(GeeQ_$Ej4)J% zJXbHlev`XX#s5}0L&iIxtcb=7v%~ZbIm`cm5fcCAK_A4x9h0^C6w*j5ipGfKNJA}m zY~a*de~VK4LxiB<)G{Ju%nn4e(mQY>EaPvUy?zFSJ&*Y7REC+jpl^~>pk_g2utj#t zaOnBkC8MG`bR2t5^<9C-1uUILZ3pidjG;*VF|-yGFVu45g(3QL0WYwN<(%6kFQ8U} zF4HrDkl=x0h0jD7zebk$w!WPHXP6~Q^h-prfj?q)>w(4$DWdf^$W*XA!1PMDMy0x2 z>90|Ahnzx9;1z0;&$j?ZdJ&bgtyOkUO*X}f0F|FT#r1!oXHdQTym9gdGcX1XtMRh~ zdyi^R)s7yjnjNYthDvnBKui);?~(Dqx4{TjEhn&+j~=XLJ6JUgmgvgiz^WO+TFwcq zm7@o1#SYdo21|71a9}MP!CJ`)tkoj{tAi!hDh6vxY6t-)7%tJ7;FXdp@e{F3zpN(W ziV?2Wc(`H~%#tp)){Y*pwK%*q!6iC#IJ{Pkc&+7y*ZR@pwH}9;Cc8vu4u{v85wG>U z@Y*R&4CP_B!l~)I={40b6`ofaE z_S)fVui9&0K78$H{SL#va`@WOf)}oR_3*W$^&wpQ+Tm+Q(llKA`e3!+B&Wq@2{sqG z>^XnFNJ@Tz(@{tNq^LOY|N}-8SIIhwE%^@W} z*fETd%z#Bt2b(Z1JkKc;SkMxkQ4f(D3_a_2EGVhEvK`rR-zB#bY?sU*#CR@idCc;9 zZIg3a0x3w%23frF-;jG`#?a0HK9uIL@k7+oKwh!^9r_;}a#|z7ow$Qi+|jc>O9Z1y z=d|(yTJ`pd;Votl<9jtNEMAOheG1^P%$!bcpNz?DqOSi7Jr|B?D%|?D+8*Rk}*k;bV z5xi2uz-Igpon45W$zLX8UExV4XYu^$Md6B6zj5nrz%a!*h66(-&8-vH37A`lxTW=F zNGlz9YoY^L3vG&kv`h&nOJ>hdEJ4u}2#$I(A@m+<1UrIGN!BxrKxzU&ne-2pJ?!l1 z1{{8v+FwVtV;9dUIpT)8Eu*b@G**#Eqgsr$B1h_7nVak6<^4damZTWs{Wu?_@ZEPpG z4yJSbdj-6UJ{9NE1+gP7r`Dq`KgpD25 zL&}yaORhYGtKWq4|1YHWn2W1W*vojfaiSpN!m419?Ubm$!4UGN0%kstjHhy3vb zS;tNUUf>PtS^NoBmsYFZYOS{Jo*}1@(-HUwoF%8x18{|kJYpV0p;FBKDfDOs9xW4e z)v3iZ(z69#-cA#4%ppf92V6tYt)U*u#4gKkF%`Sp=|KH25EU&WJ~Jv5UL^Rgno!Za z9*+XWt`bCRzl!epc#P9!u^FKJCoe z0h)Rd1Xq5EYbdm?n?6>_*hFZL?Lh1S^X%Oa?W zy$W#&Gqws5zQbOHxSbhWg$Pt(uR>gvjIBb1WUyBu?oh^7A%YLss}PqfW2+F??Cn*E zTbHp_hzsRVg(joKd50WO$y5F_@h`|P_(+!}>;zrTLBWQiDE`3EO2Va2XhoBRvNv2* zA^v7orrBb?zFe9w*y^mqe7oK!=K?Xng}V2o=Pe;+A1@;VX!uQ2{rD1?zAC8}GO76Y zOs!yDUu&hNT1l}!(SWsnt%ViUf<@)0KcPibL^dgTQ(s=6<^RyE z3%Y9F?radw-ux@JT*G?k0A)3wi?XKLuUM`RV&V+@qtpZbO&sdo-_5c9_<>OcZs~NI zG-ws;EOl$C^p|69vGjWm{Y5SKyVe6~UyTJyZJTk5VXYcYfdM1i)H4|ynHA)U8YaJf zdFN1Lv9vnfAYlEn7G5^gi!^W{Q?u&lVyL2A0D~OnlM1j`n$5S|VfK?6xEhmOvjy?zj z{FsAOZ!o=f;rasT{=qh2@eVTHzQ(5U{nmqaCx6TLH;_7>svoURGjlI&R`GA5ZA7ZZ z1yF$xrCpYXJo3yPaOh7L2}<~wVi?@1@W9{*0HzAq-XdUh`bovZrozTSS6`#VmTA6* zp;F&|I$O)4{+{}NT7A!^f4z6G3)Wimm43TKj$D{4TP^)t1U0Glz3%vXt?pj$jrOA! zRWT+^8)G#ut+(;$$0KO7x9FcE2!Y`LbsdhJ_Pcxg?o{{*9bnr{mjfV^w!g|$L01R( z$g9$7>d*Y0D`IEeB!aifzU>1o)q&lEwHPvDpeWx#F>Ny))VCJ&H;p(@%xPEySxtWW z4^>d@)x(RWTy<5^w9Iucau)g)c_YsY<`kUfQS>O?__3HO-r3-`lbBd%Mf&^Nni}d@ z@F#X1QXNT{Y;_T0z&AfZfGLz&DX3aZ)CdIZtXs`~n=}fYPmOR-X3!6xQcQ?;A)KEy z1u>;XNp-}&T!ptDpmd?=^5I@Y6vQ7;J@!SES%gVcAL5M1D8yH|A)5l3TIu?>jSiE- ztvZrmbsGfH-?DAyWKRx_=@!NmO;>~|COVKaikQ@>zK{2MzTg%Eemjq_T0dh7kt910 zHXpb2D%>CJzPZ;#*~^klpJ=J@110&$Lzg3H*stC`K<4mJO>%bLNJvtZN;kYmwZsWJ z%Dkc6yhBnnm6xALil!jnVNL*kfF-F3K*%zfu_;T4vJy>f3zGXCTU(Ia_}JQltipM< zmB{dr&NpLQ=6zQ=d2T@--JF{GfRxqhxP{MW#Eu{G!yIE?&=*mlIlwo!yVVIZH zNYhq!v0-DwWV*784O?4(H;R)m=!^&~0mp_nx3_z(zB_^cqDUFlzk!9Ag_ke{-!rNi zI6kXhkzCAzB{-=hveYV#&9oAG$Viz`5;@5xltiAh2_=zta6(DsGn`Noc@ig-ME=ES z&uK&+Sxu9QqT__5qG*_silT8!DvAawsi+z$rs6X5?o_28*(={C_xK%M=jeXYzc59| z^k)eU`ibmo3Hzx1Mz;YTBxrkz?1=g(-$0XFg#uNZN7K{^CZol`X5UHPOOd2Lm*nM{ z=&dFR6~)*`LNU8w&IXzu+X&Z^C|2|HNGOH{`IsqQPNG72y zoFO;7hfu&$5uJ+$mW}A;#F0LP4wj1OT$HeEM6Vs8xfe3PqJk<8z{e|t)QV6Be2-bQQW6YNqFRp;k!rp37v9<20OMM z4>Y=Sq4G9OfJUtYCTK7jd2x-*b#)|D$;>r+0<+Z6sMW#*4JIQmXk>D)yrE%|hMJ*~ zEoPliYE3aggUQGYP1!7KWCpQggho$KmzrzT%432ClaUuRGG*E^LZj!sObv}P6_}vG zWaI^n%n^5t(CC>fQ$wRn3MObU83RD`(6o5Vqpk7FZFFV<2E<_tF9@u}TN5jr4qgl|p-PU_}M7z=|j(0|6_$mljxwRXSj8 zq^AXzXH~JvSZTx<99U67EU+SW%Rs=&?xh7*VwDb98|nD~D@`DS11lUJZg@}T>6vKsK@kT>F5J^c#4sjN+vGL^My2d1*N1(nytAUL2dB%RTbsNtu)SMrg=0>)At(Loz4tJAJ}jp>E+e+=}9r@KPv>xzCw}N;z^fWs*$bJ1O)J zOt+#%t;`hfJ2N8x6kpu(LIUW39Z;MNjZ6%Tx-F85Xp5I-j6iqjADXVlD54lM2p`QO zAgh{slrnpgI}o+q6bc+s5iBi2O$h#E63(BiP9;<9h(3ycjbjF)6&yta;3$Wc9@TQ{ zXaJ8KZM_;|=v&XQv!=5aJq6;FlEzKFakW=w6ukN&<<+iGraZ*4Bz-&$T?SXo=&Sh&C1*j!j!+FDxMXs*;Y))J$? zzf5Mf_qrc8x=nBUptHqJBt-V|XWeU_=fASW|E|^6D(T|K&y_C6Bv_v-&BeaG6#Ew9 z3))ht*UP1gECOLfJ1CJ+ITq#8Y+{KEi6zb_mY7K_F`ZcATw)0~p~PJ2Yz$fpr6~v( zoObD|FO<$?DPAs3#`dmQzgjx&UXIH$aIIb~O+d_U+q!Ow8491A%Kiz7{lp+bu9U`I zO01#kpckcVRhHqCvDiYqM=A?s0+hq;_5%-MMfUgH%K-csDMC`CDadn9im$;hL$$)^hc=uaafT{y z5@*}}-0fG%XOdZ^(P=`<{ey0YnOK?}d&W$Ayb>XPT4YKfpSbo-hXTF z=DyNgQ;Kef-Xp{8OPBwov+rA4s?3u)ud#)c1E8Sa-P`eCtN=o{@SIbbj;yXQMENP6 zD`CC!X>t`5w$N)K*!kX*H-mKk^b8A5(%pxi1e)wOkR zWqD=!B)1A!XZR*R6qXd^?MZGGw(6YZR;hY2PmCo!i#YR%J>fgatrD2fyp!B2*dk>; z7dgqT(gOo?ptw*iV@OpgCAQ64Xh9VTCXJa-uw7JZaZz{Fh<{EtPC%!ba$zJi4xyP4nWf1vPWUAv+ zsPE=v32u=?pj~hD&EqDC-7@T+x)gia-N!pdVlOjP$UMUn(G?iBd zg%nNYl|bQMaw?SU`4cP-z9(r{`Y|_pLUA6f7;3Rcq<^7wWfbnpCuvut(6pu2NKWIC zm&{0$o+K}!k*2M@q(+*yvWtzu$Ll2R3fzwHuS*FjU=HA7k{9N%CZ>|+U>~NE${`r( zi~hw0OElvETA zR8mnjQcPbzNxQ;wDyfS!YV&Af-oP))a+SuulRQbg5}ITDZbU9|zcb~hI7zz_SvZuK z8Gx&kbCBc`@jKHPpQK%3cPDl207?DKm*vuAnmI|k0v_DGBrED6vx1$ZU5PTbQvEVZ zuamSZrZxE_?Mlick?3U3mZIS#?MjIKn2(uPRU4!MCUlBq94Bd4(w+Dnmc38Xt{B-n zQ4zE6A%9~88B$wh2vAFVh;ezt~FY?Mn;M{Mric#U#X!{J9ZN^n2fxjk)g5jhUO&gihs;M zecgp)es;CCFkfP@#6Y75cT3GRYUMF;jjWW9aAvNtWTcS+$BvO}^r(@kp+RR7hVayw zjJ%+c5!Q|o8a?h~YG}|Qi~$-<#sJV5(~QWFaw(w+oub>06i?EwTt6iRj8k`PX=wED znW=dM9jY04a7VAOhPW0WK(Vr zH+S?PzUk)Pe|t zAaBNib)V#Di3sGgNL)wPXvl%hqXf@KL@{I{Fv ztxDh`-)@EyxXcj(Iof(HsCIKDlEGyby*1t z9Jl1GuUMCy^_93KXMN>pmz+Nxc*&6n`D+WHqkaum9H9GT#bMgYajrPPQNfA>{ByG6 zte&hm|Io?a{$W>~JCHW+X126(WlP$)k}Cxt`B_T-hr(eMQEjPNBy&6yBKz^U(QP;G z@3i*zgPe$HOCvWtRdR)YP~{mE<3EwRpauC!E#?!I+mcSA{ME~|4$o70?l$yqk^Oi_ zcJk)_$Ic9U;8iWpYoX`#buuC^DDi)&L{*v1?bj8;=_tNR&WV8k5TBY|lvwQb8(WWZ za=IR(9+@#z4|$bF;IOZkAwa45YL#PaAg?f&4*jLV-bUxN!U9gE zd|xrV^^|R&8s96!zA-38v2s2Ve_jN2K2N5454-KoBd_sMy9a4Tdym}0tV3^+NgmIB zaUaII)m#jQ!&Xs3bm%hekqKO*JA4ajrsp+k6$%${*+oUZ9}}?Pb8<1(6a5xBB`V^) z!i#d z>fekv+b8L5GR13xiJ#j*d?<%Ds54qoLZ z`n@lZpC2LDJa2oi`ypex2m1$t@jh(#A9~x6*#15!Sl()_wx;cZCP+ynnDLgt{O13) z#s31s+@n^v)7pVI(`2gK+B$#(`cB^qo|ee){dS`V8Y25rif{M}Z)6}Pv-Wo_^fzpD zwD&9U_t}sN4UeFKt%GK#)9rM`2mW8ItE__HZ`ok*H*4@T84rB3MDf-~`!GMjdztlq zicF}#Jyrp0j(ZpKzg#jId@ge@(Z>T^Fiv;Nf2 zx2!)Ex7ETmh^toNr{bPa_^F?}Sg%*y&)C<9T-ibii209Yk?x>tii$J{J(G*(+Q!<<(P(FyN(M7-0VEWMu zssw8@+OLW&bFeWRCu|l}g&^beAme~c;5rS2faIVTahW_w`J!aJ#lVUFkOSh-?>Jj!qEaEx@gQ$Sh=^2<{|zkkAGLfk;}l&i z0UX%g%S=;eub+W^xYWAEX8cWG<}5Oq#iMho?+QG=2GVI! zn719gV=!uaVkd6t?fUN>+Rzr>lxt3^D1!c)0~R2( z-c<2=Gs)*$03*E^tUJse0n1XM$I0WmXximBB<@ zF~YSP4_AzGP}0TL+R@{+7KfK6xI||Thu5kRueH4JT0eTc*5mNfWS8j7;qY2B;Y`*G3#(n)ni(IUHW=M!Yr-fft(W8>(_CSd_(h5FJI6B%AiitAkbk6+!}iVaZ;5 z?eMi%?X@o-zIL>JhhbkieC=q#3)jAS_}bC>5Uzdg@U zc&lJ=?}@w_ORCfAp*q;7>LDUUJw&vqhlm*U5K(j9N7lSYp9)kO`nwJ(-C)+{+i$%E zgCKvSX!EedW#lD4?9SH? zTCK0QT>lq(F4Sf3AvO!7^*;e8n2DaK+5>AcSlCFBIzzNe#$}6ea&K>QE6+A{>vk;% z>$XGTz5+IH*EI9?Y=|k;;A3+)??~`UO>dj!K6G7-A7sOZX{;-bI;BNy{`8`_6kES> z>uo?Wg-O%mvGg*%XY?JhQm05r%O!BkWA^00a#0C&)j%eU-b0OG*NQ30iH0#qO#oj!N475JE~Q7XKDUCCUe~lV8Wp#`WQBUQ8dcYC$F1soN zXUdH_Qd@^El%$K>;HM=LmSaKkEb1~;DFup7@; z$a^+7o^-xlBG*@aHYuia-D(;Z;%)y5H-fSOqga=cnvDk1j)!d zE32Dp-pcxV%>@UX{T*0KAMWimTV3$r0>32mfVnk=AAzG(_z^f~g&%=`mb*+)j7YV< z`jO4W9$%C3@X_<+S`e+w^Y;<#iST;&eBAT zTvi{b1AW|qHBwVW{I)z^o&Oh`#`|5kF%$?x==eHWPW_$24L;l?s{?$#E}*qiQ>~Ez-}j(>BT$D6b~L@>>rwSL08S&y^0F;3D5#drN10= zi>2Rl=&v~B>9`Kb(*bonHm6Fn-xzDET{(_vZ6Pk$SE}Bo9`J7x)Vgo^VZG3COQ+MQ z_U&keZ`%GM=x5K^>~JI^7^({jx*K zQLtn$(!hmIlvFEy0;s@Q z+Af_!uD9k6IP~KpSx{V+Vi?@1JE4K0idBpxu;nUXbo$9mt%*TbU)QsJ4MU~A{dBf( zCDQ8qY4tsu{`KC$E}ZR{!*34~=&@`)BHSXVZM6Y*$KPvp_j+%DV?K@FA*%^G8#%Xz zXlE>A&rnoKII0Q-L@-h&Pic3`k!D7g2f(RqL0aZ|K+Jl1n z@~eubWv+IFx!xsj*N-ApGKKpxC`F30c5qr#Wl$X4ni3PUqDcR}w#tS&7W|35231EA zW?h|!UE}s>X2^n@11~8i391$|HUa_je^Fv!5cyPOs^e3tMWoem)VX%*Z%%O_yL`kod4U~O}twvRTevcN9MZLz1$z&cNOL?d!IXiFU z5tZp4L`Fb@K<_edD7OGmDVoa5mn=n7%Rc+q=T<7c`yTxu3BZ-iOk*2M@q(+*yvWpEH8zy6zU2NFe`nyqr@&@Sxc7q?~+}z&owfgP^{)<}I zAXtd&UkCkIX`!aoFi%J&}WQ|3WJCk;upqG3uZipD9aC>p4wqH3g=zRoPaQKYJMIG#jqeBGsVkE6e}?^ z0L6cu$km`095cl$Nfaw-X`wjAL$XAZNe1oYm`PquA{j}2^RdApNrt`EF_XNOM6yye zj%Ji$i+9WAb5X*w5xtT~bfOv-C%RLNbS`RGHlkM(iB43+;)veG zCrt9VQ5-O(JYz`}u&xZjQHv}<>UTANJ*~aSS*LyD%rcwG?ZFbTQv3DyylcL9gP#HQpXVS=<=v{eZB1e1^pCRjN5f66=z{JBK+)`l>=;1;Q#5npF==UA$vlQuG@Bq8|8}`uHX*~dlE-4q| zpd!ykxbE6-x7F}$C)5V3lyVUaQ^P7M@Y*r4C`9;$SXh7E0;}v^fCh#Y@n|5lg?b|Y zOaRtKdOpBPQ8_rUqJmgpMQp5rfR)`#3#`N{9k4di^8r@M+rfbq6~qE7qKFLytn6M| zU?o=RfVGjH53o{b4-TxTAQo5=Z*3r8W%tqoE3rxktc~=v!1ByWRv9ae7=r^VDu@ME zM5Y@ESlPX_z)Gyr0c#^YA7G^kWN=_b1+l=27=8l*E4!B#Scz3SU~Q!51FW>83=XWQ zAQo5=?QkGqW%tqoE3rxktc~<+!2W3w``6;QM$(HJ1&uf8F|rep9vh14b|a&wnQVg=kT}XLd=QgU0yh z73i-P@XbI`4cG34?+qGfv{4E;*VpcY2aL_Ii6(gAhXV>8*h_|WJA28nvcgNMzz#AT z7QenIU5NyN5wwOCTK4#qBA6uoejz$HB)6r*EUXFG=?gDy_L5;)Z7&%XKD=av%LG9> zm!aINK9{A!O{PZw3ZZI$D2Ib{S}r$Sf6^fr&DHDoTa^=mPaJ0tDCGamVxmzLK`PwU zK$iGCQ?!LjhlB{xTeqC#ls*?B!=mM`q(dKKj>P@t?F?|O)!Nc;Yk>lAQ#b54o>U|T z{|*cX!N2bmy+oxj3cxnpic+PJGlb8X1<1fY>51%wBAi9fog@?ZP73`4)2(PxD>KFW z&Wy-EMF?+ajrmw)pPF2y}=3q3LRjB1&BqIRdh(sYfZZC%Hqv zVazFjs0fx8VMPi4WD?Gwt4?LGAQF8PPanq>L`yje3&7zHt0}4rse>&%My=vFgUG^& zTD7P=1GKx1+Qrca5z|IAgXh&OLGqofapC)Q)6~%=_En#8C zy_iE%<*r*;icPHR{G$r;4><{fYyzt&;eaxI&&THh$Pi@x+r@B{^}j8UzbVuW0o7;1 z8HIQ@5>rXT$3v!a+42;{$Mq?*pi)N?`-d+)MN|Kcbh0=I1^!LCpF?&2zluHn{POn7 z>ix~l>Oys8tF^GQ+^j8ZR+rZns%y*lTPrKAW@{rc{`|{iW_zz|2tWU9RXZEYVKtbEUc1x0hnyLR3O)&E?WX7Pl~>IF!h!9E);kHnGHo#1iKd zOUxvem`*HlF0q80P-3ohHU_PQ(i8+4PP>@Z7fNTc6fc)1V|!PuUoD+>FUO@MxK^*0 zCLq#xmPn)U$*HWL)Jw!&BLgWG^(&=umlA7$yL!D;8nEs%kTRB)lxk@->n97486pE4 z%$0^?d5Q(D&cIWJEXAvkl)=sprZic=wL=+NApg2TerJT7`S4+Tr{&$>Yd-OoH#gRR zk!Qst@e_hA?Zb8x(r-c1grYl7uHd)3umjya*!9}*mdC~KL75qHR+K^M72xp&axU^1 z;`Ps%U$EQ@#xGa~hKpoIRHaAmTY^^JC3nQDeyh{l>o&UG#*@djZVSRfi6=YlUjIR> z4?W(MPk5OFa1n>#i>T0ccW+mG;w?chgEn^?{oR8dPh{hOSB`s)TorA~S5>du+B)cV zA*l*3HcO^OvHR^tuhDFJ>|a&^b;+c71oaQba{5hPy?i@Fg%68 z=QMy?Llmlhn|x7pBS$=;Z$Wz+n$VG_$y1^|E0$2`JaI)Kj3hk&jziY@i^aQq^j~44 z|4!{TjJ|yG28{k|t^VsU`im`*_eN~Flifd3-4ghF_>bQz`CfrMpSmmVS0Y(yz~}%BjujSu#Jj7s+=L?9avgLw+pkwkf~Clf2aLmcdrejB7dP`7z8zj zzNVa@WqMz#1Bf^mQ(B zLIMFp5zE#1sM(~nY2mM*g&3NrOMgBF`B46h96z%W zvNfamxQyqc9fz;V?aHEG8_aE$Gbv%ym<;nch%imN*ahT^M{;^>S5QpW0} zx)*dp6`7w&+~vB-TZ;J?eT#~=<8jT~AXQr1;`^nwhmbKD@(1O=@pq46Q=8%N=YII? z=ZoZ~IE{dxQ5D?uh@D-;rPT#fFNrtxI`wN{NWB`93RdRny$1HwxMDK1?;My;?dJq@ zU?sJ`&1eu^0E1`-ljj_)@bzF|B4AT>+kT{tljQ!Z+ltbZQNqpw

x90e?Ck;gYy{oT3Pe2y-Udxj*$KKck zd{p9Dr&*e`WpYh?=bP;8AC{mv53a5!d;E^$eC*8eN*3k59Fndyn(s)6Ztj2VToSK^ z(I)bCmZ8L-Mqbh2l~;{>Sqr_SZxO;jm2Oh}A1Vjb*9Dvy*V}NCyZc1+Qj^D!_6cm$ zzBX(>^b0v*nWI4uK?+JiuAvRKy#z0y*l;PST5)1q$tyazL;p{QxO@OKV%&*4D1i?> zr-mq4gDIbBdO|IMvns-;m8l5wx1jbyN>MR-M_IK~P#vdw54-KoBc8nirdsb2L@;*1 z7%wQ{DXJFtfhStc#bCs2yRQTrS(*NbL(X!9aJ}yEEvTPfih`@~nTve2sIEUV?$=W8 z6=SAB0ce{-wQ(NwSuq7+PG!c=&M<#gR{%V6sq6ZmGxTwV_hah0fx$DqbB!Z_tu>mK5(ozEMI-wNZ!vS2;_D z?}8t+J5I)GBjV9zasf2wot@TJzx|kLot?%5P_Em}o?9S8mF>te%w2NZv~N3{Gaq

jAvg=YY~nP<6;I=j5~pv~RhYYh%UH+3^eqTd3#5i-P@6uHr&E1#IB}R9$?!#*vc!dtH_->4*qJJ44K>TI+UHuN&g>G6MFGHSo9AHwm7>i1- z-1(Cq{SAlw@dR1N!rkK;TewTI#qW4+b*&0Uq2+b=3^|34>A*5{mYhZpz!xm?hVu; zA?i8_A_Wg1aS1oJ3K8tXUWK@w8(W14 zYhkZKT-%MULIiNIS0V26##SLh6+ji5jN(2}zIc+8z<(zG1^ERZ>9T}5ddbNv=8Y(d zKQOc+c~xr#T_=WCG)XASvAKxz&a#`i;5|D*O8&VgnC~XhJtq+0+TS3(wZB1hYkxD> zUHm;b?Ses^zE92tVt@;E?+LgRBU=GpMl!2rWia9z>c=Ph`l_T_uw7&bt*87-6}V{_ zTEY8ct(8H1wTvy`BY&-hK^&WInrOf$`&tVt3Mq?qlm3Jjs}o(Gly`cy6Njt|x@z8j znh?$l{wuaD)_PD1mBo54%9_eNOwIREF>%38QuQ|FuB8liE!6rQq^|fOcg0CfI-N$f zZ$~S9kn@XLS^=#G^15v-P|8q(lUHlia0(0<*`^*lh6_!SS+nh zHwakZZGaNWhI)|(F6HEbgY$+e$_23B1lMo{7kWp`Ir_t-Nn`Z9p&fZr2Wz~(K!Sr52KdyyG(6oKNXzKGMi;g}B1HggC zvLeUS8+4xc8o;}Fzx4pDhglCRpYl(M&R?Gna(EMVYlHvCuI^W}^`q4R-~1bS9GMP* z&|1ybxB#k`QwJRS(?x<(iKZ9^cj``PV5njh{p1vJRlw-Jm+;Pc8NR^k;vA=-z|cgRQq0c{JmCpulELQfvJjK zV%iw1d1<}FNk1Mzo4v*OJhbkV|Nc6h6ob`!pAN9?rfS|NZGV-if~t9v{v&X9R&zV) zGpW~C5j!VaC(u$I*gY7VgiIu8i3*Bon`z(JK-UzLrC@NyoNBFJ^w&UE6PQwaY`3lo zs=azpP_F%vqG_4Sd^3iHK8P@T5iFP{t;F`2`ygGTKLwuf3K-q^v6w27-r%;Am}7z> z{e5js4RtK|6Gzrn9Z8sMb$--@b3UV-T|w1C#zr8}Z#mecQAjuBrcS0GKBcfmyAaM# znu3_pqNK!-U6~GVJwWNYYVSovLHq$#))!F%8UHU|U`bRTVq8)1I3u~il^@t366qOt&zmXu2XyG0}%X^?kh8^98pU@LU46{fsI2 zmh8M9vQ3H8B-1BaD*O;!KJtyuqX-&SUorXsnZrXh$=R8Ua(a(yDLHhMc|*Cm4yI@- zFaN<5O~DnMoB;d)OHvbnkiRlxQVhbwjgcxv9$&Hs`F|qk>MdxbjG&K z9=YY@xdj=Tb86}XY-!n&2TDR%%4rO#k&msdzB#M=^v``%5+Fqi`B!hNx%p|WSk&L8%=0MVENQS-DF_XNOM6yye zj%Ji$i+9W$Gp2*-lHIdT$ybunYl(!jh7l4wOW{9G z;vw_B9tR$JYT490qAU_7JaGRr;-R@d1gh2&(ioFfn3N3iJe4HSVaY~H~K}4g;-dB-2$uZUVsLM74fJK ztG+xY0Ba*XA7G`Z92{6tK`gK$^29*E%I>8FR$`ToV{N481FV#{g99rnhy_-};}{57 z*}b&DO03cWYa=}$V5QI=99U67EU+R<$w0u$?xh7*VwDb98|i6*WfAl2GFBQf1_xGD z5DTn`-7*lcvU_QPl~|<%)<$|hz)BOy;J}IsVu2MAZ3Y5Xb}uck603B;+DOj_SZPNY z99U67EU+TZ&p^P+?xh7*VwDb98|m4A{nH}8ljR30;#!~Z$#6@;8w2wB8T1(0iAawP zMRmK8QhCsQJ6VlamDZa`wWhK*RmxP>rX85d+7>)hS(^+um9>c_6cz^$gMT^Ce-?y< z@;$Rl>aZ`yC9yz%wLmT@wOv$W9%r;s3OLu-?t=%6&9I3kcp+n7Ed~$lCBwR%y<}Kf z;U!h{29bG?oBVXVB4`aOw31?6)9)9eb3<}lWNsKS3u^*)`oc?_y<}Kc+e?Op4=)+v zGC`1z8j2z2b6G0fWbEp%5ULnWASJA^x!iF5NrzlCSFhi1RZavxaqO#b;4}KPxh~e# zKt}mIW53zDjpr67c`%E-<*uYdA7YM-v@^i9R%=VYt;GkxP2I5Hcv5i}{Od9d-9`Vt z(lSVr^%CKg@KPvTx6hdc2zWgyquYw$6+L&7OyD~y^bbt8qD8Iv4hd@67q`5S zpgv#+6lX&t69c1ei=-mj;;i}RrmHcED8>xJUGWIWs-_;L%%0>9?HZ#vBPxQW zMTnK)PbT5~xoVGOV~icqCq#vWM==S}3XY-yaFoN^iE24@jD^RjRUAVQ85U8i7ENb> zcDGTxII)CSw=yeyzwVDxxgj`9<{p+d?>` z_n#EV9~JUXUSO^oGW5x)7TPQan^%CNtZ6T^{5+h!)>z?6%xz-D{rbzp};u zUSD3TxMbqN9=}P+YJubu&ka?k2+Chno%CRVyW)n+XNGx$avBXSbiRr`==Mqb}2_@!AXJgP>C{1Nan7B|n zlcjjMG#T5wA|#_c9iNQyTD@ADfSBF3m{3M3Q7xUy{t1cw#2`YhK$c8OtfA_l zmf@4J*iWMIQQ?!(te<2|sySC0j^!X0a5}B>Q(u6jni+~$AtbDQ@Iy)+kJSt<3}tA6 z{P&dnj|K8+oCkr++SEBR*Vk4yToy;0r`qwBAjJG4nd?GIE?c6lXoActxxzolkQ;MB zeqv4i1g)sFm|AFV2;I$dRh}~>A^F&uVNWCZNBrmE>=g71WJGRv`9CzkL1~P3VH}Q0 z{8e&J3nM7<9p>a_Fho5vW2kHRKK%;Wf(Cf-|_Es{7NZ)!?a`d}$>!y-lWgO+nhi z91?eaD2FzvGhrw$hI^n9q(z*LPu6*xoc62AZ8*AQOx7s}L@-W3ThLc}UTZseKER-S zjhKXVB!gJ&aubv9P-N)Vn2~IO_7B2F-8+K21`8^ z0z6=J>Pr&!<`!V^StYNXfMdLHZO&U<2Q5D6^&-f&nw)m089` zoDpyOeS>(Q=lRo!D(OEDv*;nR*kKc$*|=7Dj?AzlAR0qQ=%Jm5OleoEn8me8hSy%Qi9)l449bULb4B^^H2Bd1CBEnCbwAnmIGVS;B8gg<3K*s;7twQOu%;~ z{v9(CTu?8U;bjRP4Z}-EPl3euMHlY$8~xVy!A@x2&>t=m+dJ3xg8D>LpI{pFa!LJK zRlnBMugj2e7A2vbX8%=A%OtcTq*!wtu=m9#piQa&IgpqmE`uy&yrm;fp*TEJ$(Ts? zO_oUZbGKh5pGjsFNc0Dr=KevKXRvJY^uUan_IkZm54fnh30Eyk{!)TYY|54)JG7{C zAomlNbhGbUS~`UEx~!6Lfnzh|7KOaXPlBFjkX(c1;H+h5>iS&}F1M^kxsY@eIVrDN zWXibf&k*vjE97@Z$e9lxws%_I{k`TBZ*6032~3Ey;*t2dv$wYoq8y}ks|(rA=E)WO zb{8_6?H=rUZFt*Yxlkd^nL9(yiZUogB|N@B&P5(W^1>PO3zomq_yx;id6CSBsxC4|cpqt!}5a!!1zP$W_s%e8mh)TL;}P@D?sMOQuD!`|U=r(QJC`UseHi z$)tGGmi!3G;-<-z{tU1(J*^p^F3-*wYG7&OqV+=ZI2u5$AqrK$O};3)kt3eax1c=@ zP3Xwe>6N24*mMYQ!%NxXlc6_fuo=@Es_iUth^eEVvMkgkHIdv~}z?8^Lx#$%u=g4Uv z^S~TK!P#g+bm7fL|ILG)Af5P9>WopwtOqaVO~znhYrTfO z1`N!b_K(|%KhB9bw>(**?4+RT%#(2<;{W3I$A8V!@Bx_-&B%n|I;FXM4jpQJ0}!`v zXn9(ENABp0+*m>U^&yR?T9zp!du7m-%3$E9Z=Hf*;1exKiTNibaz@a?YX*ZImI#!J zQ!LzCFsBvFlK$YiCe5DL#-f_e`>1w~oZf?@RdjoDZVGrKiGC!#yBG#Rjis+C7ig8< zm+ZizYyqbIDfv+jwb@2p z7xo~>&_ss65kLHS2IL*J+cP|X7%?QW^PIk%oqwo6VhA*>GpoVJL$FngqWzpB(kHgl z>770tM3VJjjK*EbXl(CwdtV=NC!3((hdftuE@S4#(I{pqWIt?2SDkL8uXB+T5^OLO zO}PdiHPCJkT!g;L|<1s9GeJ@#SezWj9n+?=FclsE62hG z$zxTv8<40LkyFoRc6qsgE(f~`>kC9le1RYP&O4bpsd(u{bdiOq^32@M-Sbx!i=!^V z=!TEh<1${4?%KXAuPckHHdxz%>_8s3KA&n}wiLcJCc}IuQazT!qwb1{m`MBwk@~O< zx0QZP$f@FQfiz{Or?=fppn(nBOiz)?7SoH%9VkEw+igbHlq7|TjKcL@A{Q+7ah3+K zxhBG@j)yjV1M@Z8^~-pwDHhi znui?B#b5lrQCw;>i0tl%l?|UKuM0a+_?wPVXI7(3J}#8G{G$++w|F|r5*XQ$cq6+~ z>n7N;Jwe>+TUp^vFlmb`CcAJA%-Gx(n%BT$&HZaeM;Q5f96FhAIltwDU#IvNV1maN z$T{&9KMcwu4?-w=j+~Jw2n;~AMmq?_MTVkcr`6i%vpBreXM%o>5M85(J~N9+ zvHa*2U4pe4?N>~q?x<-nv*^+lf*QUEGLFbJuG2yY>>TteE>i?4pO+I)2zv?GBr&~|(yOU6hwk9>rjE~>vwXb; zFw(2Rxx|KfOLS#8ut3HxaieHvEf+MiRz?rj ziW{tD43_B1a9}N4!CEN@tma6-+QU0m6N5D?IfMWc443Fk@Kf47s|qIN!9rZI!qtq2 zD~36!nA_Is=t7*k+wJ5ySMvvE89A3uWB|0-4UaMBT){4Swee`&( z$KhpcU!pU^;k9PPYkdg3&|+WLmCKDq3(9GZW7pV8Hr$oh^Hu&OLIQqa$z6M6_}amU zL7lK+V3vok9i89d+E<3J9UXY#+MC1Ij?Ra0?W@Drj&{>sQFj_V6WD{x~Y}b3KMQ zD0H(J*XqOz_lSM0ole}v`B?u^351P@eNbj}{Q*zfu@v&MKCkA39`fUNv@SUvst?Gi zyWoh^`$i9prNjq%A=G7mOB)8H*RzAC^7KH|$Yih}VIyT)VuG^MvLVo=yW8v(*`_Jo zZWJKhF43O{%U`s18-{p$E;K3h;A3$&??~{IUf$0}miJY11r}_`SAxf|fD6;9zVJqq z60!N2MG?lQb^DFC0L2s{O@~|g^I(A#SivBB7$p5ug@ ze;kyjlB@7+=g|Del;~X4W0hsI@M9fz6!!P_fH#TkDX_kwAzh^IMR&^iCsBchHy7+n z{obJ0f7AngV;jrgUx$Pa}Vmm4)erh|AJwRdbTB>fhK=`F;FRK1+EvP|IsfX zjjwvarX-kuHlhpmoEo(GXF}?HC~BUt!JKCXR7jl7Rpp*raBdbCI&@q-Bs?~Y3*|a4 z9ul9L#f8=#7Y_*#&Ei5GkBf%{Xl8Mtr^m%Z@*}gjP~hX@A@PY>Txj-j@sRWYhA3p= z;JMV~^WdNx43URK#GED0VEnWjHh%gNc{h3_a`(ZMwoq|)P%(k^dYkpe#^y?WVY7XI zX<=o(*Ij6D^qLF3wfoEMmDP=Queq8OSnn>md>`EPERpp#yfy6l>Ibp&d`FnQ<+bG{ z9}IQU#@c(Ka>9F##7IdYT{F4pAHfC-yFstBy}1p%Esy$l(&M_z*5|>F>a|1zFj8|= z<9QMNaqA^e$lZ>qQW%Sos-wz^C)Mt)*TF37mDC}|;wltdCRA7|vcj#_T`=T&J)1jh z4B5goi~QR21mZD2g{O{lzBD!rn=WOI1eQtSIXfxD=O2wd*RdJ@?BcEI^u2z2Z*Z_b z=+V@sKQSgY(HMXuuoK#M+6jq+f9uD=N(cgQp=(s#xJF6-Wo#o9XDt*qk(knUA!8e% zqeB()9imc5Bj>S=kg%XhBN?!{rB2YXjSxF1q+w37A=21J2;AY)neYw))!0Tz8Ms}% zJyv8;{+3*_M2t~AM zD!;?nMktgPO9O-Lo5nUm1MpoGDM6|fBlBZmBlPQKq|MW*I>#155dTTr{fyQ^C@xa- zO$vhBz-S1W56uK>lg!3L=au?4(3EMt`Pg&_i+@-|5XpMX+&U9@1%a80!#1fn)1m)6 zAw5G>9Zr7~hC^xXAjTF(NQzyIh0)K(Ss1x8RQ!0^7k$NQU*w=oZ(?aDB{-35DLL2S|5dP`OqA#n=b@DkcAMsLW-}M#ya7UsK+) zmqU5Wq=A4wkKC@(@u_);^2^ z%NAL%{kPlw?e_hh-s=a^Ojz!)cfv%GjCaOv>CgO|RdNHHuR=bPy)NVo_j!ihm5t`=IAPOpZ`gr0c9=pl3nK0VaiD(vZ zeE&)|Li|^J|Jo#v85xz#R6q(khu@@m8GB5ri@aO2pe~v)0?2&_R60gOIE?`WGhjZ8 z^y5YevnzqVj=DtPu1?x=6x3!35djor_UN)LYr(%MktXWWTOyMf$^-EtWD*kv%+4y@ z+>0-jRTLAq@tjkTA>yf&9q}%Tf-Fsi?jAL_2l|XJ6miGXt}hIxkpY#vU;6@hFLBM z%H0$5)H;HY!GDlGWbo`qDo+<6ER$yG^hu%Mq#EMXvbVP(>c{qj-TiHr0!lSN&3g0X z$)s7QdEaw(@{0mdnJ6n5 zcoC$Ph<0+Kx<{@fzTxc#oF*#mBWI?P57N|ZPnE@?{tCGkZjU9-lWh(gn7Ty&GaRO& z25fBUlc6#u(fXjg87_xA8x$J>@AM*>Vky@Z{4Pcqj%k&h)9$<0kr{#)qe(V0)g5~i zxF>TA8F@D{RUg&*OwjL~ASZZ&5brF{EWOhk4BlZ0km+M0ZwSuv5sZaQ?`fe|7nxzu z>aTO5L2jx1P|P1dl2HSfN{K5O%hn+2dx7UBr^Cq8WanvvQY^Lt_w;ycpL7wpa zqr-!v!vXMMXKIjYB4h|%d}f^#a#PkvxDN1i^8P`)-`?5jLB8i*AYcLrm||J3y$f%i zH?v+ZuP?D5!Q``xe`MeN)#bH~8l(*p#41nN5fFEkTu>cG*|Y)ND*c%fSwfTH?FLNU z&O>y+6yHa>$Np7`%qRE3llvr3e#kmWMe<9vE|%#=awU#^GbU6(G&Y|#)iC6ToyL1D zzddY`OXk%E!tb!2tF0#WZ7(raU*y^+TPo~kX}P?rCW^aRerqxuudflhSw5RQ#H~Y~ z!4fVUp?jz7ubMAQ;d`f|2VLnxH&a!jsC#EM=H7`zC#Cex&7NQfGK-)LJ4VKXOh~(< zI%&o^Q%Jbch0Xi$v+2U-p)t-XPN>o^I}BSxbSsZ|31ov>){PZ;I%UwU0~$t!`s^d2MZ_xv%T-}*8fx^e}2MXf}0yFE8aDEv6Fdv@m*-H?9G!K{Ep0r!97cKL8vwY15FD9@TxRTt)3*9ZFrB^^ z?~~I41Edm?IqF5Xv=WHQl((w8`ji!~ZS1H?4J$=xa>R<#ZE}r{Wtrtsmq5sJG5SaoWkV~EKI(y*S$xe4TEqscXyKH| zjbsQ7)uw95&{N5!2#s@X)j2^Y?@LZ~P-OU#=E??Io&kAB?e>NiOLl%3UDiVd5<@_0 zv_dmW=(Pk>v^bP@t50mF(>r}Qh!CT3S27yg+~8r@Xn|nZQ^;LlWe9wHX|xX?IU`_CPHKJ10{cT)tjKDDuDz^ZvMPN zWmaL~g5-WL+YLxm`x`Sn2`e|In2!-Py1ZOKmxEn}^<`OgSsd?{I~heihvqZ%{8h!` zsJsg(H3eFa%Xl+!cdjB zR{Av|0S`29ij+&v8RJKq#u+IhYbv`?Gul8C4F$rgj)yjVN}Bna?fT_ec|jHupM&YE zt1Zlns{&p}6Ixk{HzUP$sw)=G-z3$$ViKy! z{9NKK*Gm>D%zr{jROG!w1RD43*C1WmxZ&cj37a)znGpXbJ zw-z8}L5AYjz_dHAXOl6%z5xjXUUgAL8jcJG)RXt%s8veqCn00Ni_RK9;i&PV=ZQs( z1)*fnp@vK4g@d`1i(^2e52RjA1p%Ns)xPqVqlf;*aS`A zlc-79I1oTC!`X2lfN>yz$)yscP7E%3=$`Qy*SfGj<3Iohy@5AF&a_yrEWzB-uaQd( z8<29C5P4_K@AO8W7Xy_-yZcCnRM)gey0aUi($cpBSi;{laAtsZ_n3Ri;<2* zgEqvqM8|;uxQzgF?-wrq=finMtG0w{KcYYYYwJrZ<3Ir8KmZ4$$)on?e_SGeJxwy^ z;9FZ-srze<`WhIfpF#RnFr7G0X3zt$gor$1_9CHD%u*!u2n-;+WwQ97-|lYr_6`TW z&Qt0)eD+vK2Ec$EWuu0do*_5&ODZ2TDz9?1LeXc*Q_-SzTsbPdNS;N7aifkGo*>ut z3!Dq~ceWq)d{pNuY3L&Axu{NkV`&o)`f!W{(UGC!Yh*d~dm1-*af38-yvRqfN$ROz z1sPALeaK&Xj%?`H0vZI3+k20C{X>=~=Loz2m@S_!z@CDUP>npPU)Q8y$c2Lj5cdi4 zw6&};q9V&7lwKgu=+dggqTb#8E)!F661ev&bmRr{oGz}rDeI1-q|Vc{x~q`0Ij#zk zy4799ax$JGovXWwmE0#YftS46?QpQ3R z#va8D7u5D7XOMa({)NQw@RGTg9PEIH5u03U(utz@1xqV(<{7Qjbv|Qg#n=gDIo=GK z-|eq&55RV8npBk3AB?lk5~9AbHEeu`-P`yMTetC@8Lr{?!DNl)xBLpZ5G)_Qsrw%c zAWL>|9ijLb(c-1l_7GiL{aCMMo=UofOjhFW*;>I2y3tA=O9g8SSm9-~kVmcmx{U^` z*D_i_7Vi#`4gIAOc|wmZ4hqgE_2S*F_5pY?@_%U716>c4Thch$ng;Q8rSkWuKpOuS zCHlK1^0BxML0}TzrBm_)y{1dEPi$%;kCmGK)j?dKEz-SBJ#oKDQ0sTUcGT;Aqi4Y_ z9Z#d%ccT@S%l;O`fzlKjJxBwTR`F5Y$3>$WKJpffTuYDL%Oa7x!OqXpO|c1@Q!Fh{ zmj^7+wN;~F$zG&^ORG4k?q>iOCTCc{zzzDW?&=HO?j3K6{k6^ZrJynT3H?rVwo3|w z(hR~o;FmY4v3!!8^gBE4!2rzDLC3SR-)TLst7M<}&RGZ1wf(L2V8(a_3-9z?9Ji5EH-RZpq)Gprdf&Z;Ck8w8?3PzLYp7_&<9Z%Jd9=OXE z6B1{FeiL0IQZ+7s3Vi!h!{7><{OvMXh@TXpVen?}hMrJ8RjeeiBO;jS^wXJI6N9eb zZe@ELhDyKtnQZS$r1kq3_4{o3w+2UGfX(zQmf$`s5eelyskUyIyK#7H2J;zvq@NB! zZR-oLcluqX5_%Qx!ZiMdtQY89q+=(^REvcZ`n?nA*0*^3sJplK=AAoYEvj#~|Z z4L_kd?%z~z0Kr|Jw-jY4$ren%AamMD&rfK0!~>neE5OO3+yq6qEOl zP<^X5imF^iN&)JlT_>G3{VX(Yi&LVxosZ6@nt^0S{!4VrsPag76lOJ9v`W zM2Y@Ni9F_N0K`FFc6(%rFw%^;wU*>NNQB)Qc-kxEJnK=LBnn0`v6JGLp90O zd7Bvx#%y3A?%B>Zz6W|Vv=NwH<_#4V+apC&MaB3?(G=LG7X;u}lK@=F+|(5nVD{`S zCdQd!$7*Zx!wOAot9guVtsY}rYsIydIN{ebw`E^j)#SCcQCL%7#X=bCGG02ywgQ_r z_sNxNd#b{WN5Um5T}5pgY1%4k*GSV=_HDy84U;j|@i*!`gW!Z^ejZeFp|;a7I9M^8XPqsnOClkz=Gbg)`$&G7P*QtmB=u7}W+atd zO-<2S>XPfJOK#|rnn|*vR7V2Uq@qaEkW>^6Q&LehPDw@4KqVE`BgOc2Ci$JM9Y$`? z_sM%Lag2}bFa5hx4A019uW$ol|4<})Q#U8aJ&MKi+Z|w*ABh;v^@W#nYoD+2^vl*E zdvl*2@6mrSZm?!wIGK!bJ!X3ANsCoa?2%9mN8vG3+(@EWug{TC3l7TxpW|EspB%@v5t~MBwWVl-$Gs&w-Bx^=v zG@}exykjPLEs12!G36mS`GkU-<}s7Jo?n6PX_uOt$k$cDvj-6^JbA#7MSqMM0CCo*DjMDJn^ zlT?j~n{Y}UOR6j%EW-zns`wj*y`IrtL@(!-cd;wHmbxGVL!&}YKu}Dp5^qFA6VrsW zD%`bivQnn|*rZU^h&6i2Qsiu){El&ghFXlk;uf$mSg?sIvYrT4S|~KaQ$bl=rhFz4JIQqG*$avBRpn~6B^lKR%rCeV}k~hQ4}=76X-ahku7G0MvDq;&|or( zf<|~a9Vaxh#jMb1k%A2xOhyi9wrq#D@Z3rZO-i-wZ(2`n*;H0cw9LVV3GTlIlQ{m) zipNGK4sR$tWK&u3&=Lt79=QJ*@i0suWUu`C(kKA=ZNnrk+*Sk;x0X%tbx3)LSio?# zF3v7)j@U8D#_^_{Bo0huTbVJ@;tVX}3NXPW6f#e;vAV-y;$m`><_Y3*ZSw?^PzWZN zIZVD84Tp(~yG_Cbv9~r%FbRcVg1N#`W8z|KlQ2Pitql`QLLr!7c5u{~xcJ#5Ob|nB z!vvF%5tCmn5#&Rs?c3!R6|r!)8Tk(vY9Pi=c9MR#gl(20-3uOIQ-6ziL0Be-oimoS zHB?ZM?;_kZ?RU#+%1mBEU9iesE40^So(^2r^Ma;lq%*I5fY*~d?ibC3(v?%J2~81oh|*D5hwsRb=&Rnq$4pX z3q6MUqVji&I#KbTO$oQfD@vC_#t>>|PS&cCg?yGwW6u=&J)2jwsFjK0l`$i-Pf^<~ zFC?fBrUQz+VUUS|(cU7dh;|q(#%$;%`d2pB7)_LR6&V3p_0*%3*^<0Of7%+o98nP* zEn-^s8#B}M?HGQib&e!zMljNT!ZE@5unbD*&24T0?m?aWVLNqHDRB9x#fB3;uw)Ni< ze}00Op3LxfqQ(DLdybrKA8h;Ey9YbHT^2aod)9kqXK=XdFWui*UR&#Q7dqYLjfIu^ z()vPssk5@sS?{g2o6GffW35xWPA;<75Bj~$UccY#`rY0meiC}+m)>}*_B^??x!-@k z-S7IFM|&cN*Is+K=RM`!^nLZygM%ZVH~Ied;g-MI-q~Tne^|iq)tX19AMA4{OIFLk zn|ykrHcQCi_HGZ7GJkcy&z?Xotm`our><45#s2zO?5_}&(3M-NRjpiQaSJ1igG%Ky z2_@zdOI%4TaXGQXrNj~!6H8o3Ea4@TxK=qIgVsW2HcOJRh03`s#jBOGvArup0=Jp? z1a3E5^~y9v`pyz*6ka)#^_3Q6l3OLY7>m{oNHIr=Gr(P|1-a$&)m;TrPGu#f4q50j z0+ul~-L=Y8?3`kOtJ5+_kmqhPOYu5{mUUAXifkydgDgNYy7V0)Kc6e1!t&bklE1vM zv^q|xz)ag2`z+HtbXY{k{@xx$>>npoVA(Xr2^Gc(6@r_sKTfDHPN*>8!rO5|g$tI; z737t-B%Z(`4I|ybIH3Xy*%J<3I!>sNCDH9Tp+byKJe+-EoKV4tGtWe)aY6;5GzzDQP)G+l2a1v!BqmgF3!YAd#o*N+Z61YhkMk&q zJd08bR7cv}aUKPggi~kn2vSIBZQpwlvH2bHDo=*PAjLTruKu^(BQSr~;OU`7FJ^Z=DK5v{gAw!lB zCx(Wbyk%wiQS$ot3E`Z#OC$nqs{L&DNHYr>)OSCaO<%EJHHvP6utCd6!`4sQpyIcA#77=^^_ z&6EjboHZdb#`;)ujB$)d%Q$O-@+MMZ6RsQ^XH8(%?McGZA(Q%yvnH@CX2GG!q@ccL z{TyrAy*6qsID0@I7PJSi}poUe?tCI~6A9Ta;%|<#CKxJlJ052AyVP?8Db*}0@lLu;ESn1qa&xWfC(T{eFp zUr|A9AYTzrAjWBNoHap6ZT-NLK?ZY;g>{@Y;f-E@_vo;FxV^uZI?IRHYh!^OXHCch ztk|$)fgNW}$OEj{^<#k@XHCchtdN4l0z1x{kOx>HLx}};oHZd2utJa&3+y;+0<)+Y zXH7UfI68z(0VsTO_-ZA;2NwxzS@R~N%SPhai+jBa`L4ovi+{V@yDIR;IBUY;V4O7} z5FhFb&p2y>iUA-kOPC{!KhBz9Sa&!)3QKgHHDUWuC4dq0M{m_SU5m1F<~~-pQ96ln z)&y}}I*bxyjOR1Z7$-~k`k{Bun|b&X?FYU7n@5meVYgR< z)Ce|84Lc8N%H-VWU_=yVAd81-c#i{60ASvi|fO^6MvMc5qg{T#>VDKePOeG ze`#T5z1Ll6Z}gfAy|w$x?UmJycCWcQ($okL`nx$!jR1j1tCewT1hg?+BDSEmfYOgm zYJ@tZMsP>mx9l$E#Eb}a$cT`s%5fruP=qfA$fy$`{98i)u|z)W!$}9JWO{oL)*liK z4SN1g`-5KJ??6@F1DtDunL@7%pjYQ`1t+p_MqKm?YLg4 zKtb+#Pp5T0q`A^BluF+&UA_gcLzu_KH{tcSWs9#Le7kh%wf5m_M?0bC^pMzp@XneQ z&;&|q@ivZzNkoZmlar!5p8S_e(CBq*H(T7?9w$Xh8FGtVCmfJOC`>{=M&TYJ?-4mA zhrv@H=&T^DbVBV0u{S;bmtOgb9Ta-`E7plonlVqZ}Tw=8_cYKvxu|#%f`K}sE&e`agrHxOz7N~vQwqA$yFR!G|M?uhgsLk2CLAFJZY6-oY6jr{oua_;@D?VX-~f4}>Izp}oz1euQJ#3S){ zXCJn;&OY22di}YwH&1ThUw0uz>F&|4zYV`~^MV1CxkS#3G6y^DPH$^}r^{cxLM}ue z`}Mg?p7n+kh;;HQ#zlRP2Xlt0yB zkMA5o*0Q}r9~GM;7e%rA@+4sYvI?k2&Wc~QW$L@RGQCJ<&1Zm>?P9li{szt|Ir>BQYP+0reQL(~u9zw*Sy1pVzv@=WTkc;`ib zQSay|rZJ5!O!|E4UYdaEb0_6-Q?6YgGYWI%h*+YKE_qsX;Wf_41f0WC>W;P8VupiP zZVJYf@=?Oorz7XZWpc6QnE|lNl@CkwXG?NLE{aY;u3E*=ST?Z+OZr#xGwD^wP(HFTQ zRQ&Zp9jcb)<;c)Yk;&GC>C@^dm#WjZPJvJBM5|Pye^Mf61TDN~2yk$)vwcX#DHd)m zn9~YoNq_KMlV*=GZONdS_fhQ}IlTu*tLXOR+!SPxk?2QwK`{)18cSbOF3>8yFWG@b z*#b=afZw#WQP+5+ZevGHYPddmWw^Oev7&UFobR+OvphL*-X%ng{=N3zmN&)z%1LXY z)yP4~12wbQMX)Yt5gV+cg;OGq_C?mTng2O8WSgoXSavLHw%7^eWPrx`QWT8bBJT@( zkb|Je@HgUzKhJ=?qjr0S2M{BMM0TFjm$UN^6-W$$hIM8&_;?7mijN26RW>qmryh|$ zv7Juu^x+^vjK*EbXl(CwdtV=NC!3((FOiy@%b2-wG>TaY*$=tUe8iUfb-Izh&P7g0 zu)$C?zW|x->=yI^Du)aXdxEJ`b@4S<#lZuyKL^-n| zfe#A}@9z1lip5d+w!Mn_q(+ukir1sNwlB-;%A%?R*0!!M;nJ84^PNcbI14@6KEy;! zB>sa)eK-|cEB%^~Q^nr`5#mlyZ@ZVAGscfJjT6pC)|4crR7<@%MH|3*<`GtPJhbT> zn6KHcU!Ij0oL`XC`s!*6^Wv(2SI?ASy;+JkBE_}ioTd13q`3AevlL&66xZ%nmg3Dw zaqWpT|4jH6?_Dto)ntAy@s{f)Un%B4p(HBCjmNWJgLG-*imyr=5B;Ti$cI+^#orsn zrMC7oxx}RQKmo{KL2%E$9QJGq@B3ki-sj2dB3(WFO~KQR78Vh?*9$ zAe8*zH2K4_lASV*k-xT9U;D9;od)vVo!$T}Y(-K#k>?J?rw#H+zGwO3Z$O&wgG{q~ zwAXEeOcODXOfs7F9hN*g>pM&wob?@$6k8;#1eij3CwvtU@TS;bn_x_`Tv3>kxs_L9 zSSxuQWUb=km$>+a497}_o$zjWC%i?T&%6^(&Xu?Yj{5foY<-l>MfSCmWQv!&0sCJ3 z@0En#PsRQ|-s_|irqI3azkG`PQHk8Z`g2f~z)sNb`a6TeU4M0BWz|rRb5%Lm*>1t2 zRi8WJvrvb?XHY|gl_c=ALZ+BV9~Q2m0jFmP(flL)ioe6wV0>rG9{WCdWI|DcO_++pgy@|sh&x;ZZkG6tv=q>K*`lWQR!Do^e;1GB6w%_$F$ZCx3ys5)EGdtMoZ|^ zL5Z9TkrCR3%!44bjVPjnk)+j7Z31HOzm zzA@#g(3};bo&AFnIbrB!=Rz&$VU4kO`w{430pW-i_AKvv&|?j8YRT=}USqJnGR`$Vo#ndYbn=XD7cX5S7Z%VBm!sl!$iB zD?;j-x<{@fzTxc#oF>Y&hcip&H zZm!gEPoZn5hPW$*Y%@?YQpw{)nG$)rUu4SPb7ZD*rd%#xDwWv(OSQ+zRd8u|AB=*J z2HSfN{K5O%hn+2dx7U9FSyaKG`Z3U?u$j8w-+r*o93TYg1Gpcaseue7qyaBpds3?s z${i&9{k@$J;FEBt5KfBF4Kl&|2;zvt-X_?2u=E@|J3X-J-30>Jk5kN&*}L%Oc_p3a zmd&f{t4r)hCeOw{a{2b^dUK-&sv$wFQtyv|xU1xX>M*jH25_tNXG&xVO@_A{Fm*c* z(fv|J4J;=Tdj*_x{+LoW8bW~oV%F|O-SxlwcqAFGL83I zF4@=$$(z8m1ZGeUf!5-J( zJjAU-Zlx3vUn%g_d|3+LI|IlPOB8kQtj63sX9Hm>usEZV9RKfNq>9?X!sARxUqzL; zWX!H57Fu2<=DJ3Iik#{TXK{Hf&(AD&dylp|z1HnF-U4fv6bx6Xc1bOp>01OA@e#A)HJGI1@TwK-FV&By&St$#J*BHDkjh zv*u=wnqNkJ#&$=ua-4%jr|Wi%@%$55DdSzxNC$NoMH;Ax9#x~rGAe@1c5p*0s0cFE zxvm{1A;PKjq6zwuK{5I$tQhT)PLc*DrRhkV*+}gvR`7-Q|simHN{9LVKyRvd~%Yt+kuW^>$;e6BiQz7HUe5{udy`!gD0fbcNh- zm`R-UxLB$6d9V<9Ezz`v)Ew1#UPR2+dI?V2+c8xNGnhDsHz5!N54y?1Jm`SE??8p6 zA}ic#eF;odzCdzm0psWwe#_*~|F}f{dYY_ZO|I|X2S(_9rptYBa3u7(a4XpIH-WEa zKly7bYfJvxDyVqRku%8h1GKg0$qae`{%Mg%%xoi6iuvz_9)X6|TP7${UhnG%pkM~~ zkAaHsEHpHR%okpIhTLQ!nzmU2R)6rdqh9YDJ)g$|6XiQc-Mzj3-X0WvmgxKxK~Wi# zJ5qR&=vYvuiHxK6C&+c(k#F?+`vZSx`(e*V9atp|T|_+>)v0d`Z*U_%(BZA2<7;F& z^?MpOcyWU?bG*n$u}SKwUxh0a&&&!fK1Vk6Yx;;~&tgMyGPVkAk80#e{kkRvLoV*s z(9|c0j{EMTvc`z=(#-H+f#`tp%sNn~lEs1J>RcfT9eIH~r#qtQNFBl5U4;z(xV`tN z*FR)r%c$IX#9_iqI2^r59hckPi_{UQ-BqNHwd}4UbyQ(@6{+Lrx~oVXDHf_Q_9!mw zSKE`EQ|g)c7xpiB$=pkr{zNW%F%3mg{DP$wIrEHG>M}Z}XvNqGWjVg?LV*#%qqkVg z8f<9;j>HhF0%Wxpg5|?Eb^il!b4KhpK1MW~nOgSTwbhTk4a`$Xw~&dsf6vwm=JJhJ z@+ds4EnsfnXd#a_;dL7gnCmxMKo=DU%TIr)M4r%N`xE?_{+vFh7ws_K8hRWNCw<06U1&cZ%%S(Qgve`d!4Xn7F0m zY0{uo4x#m1V8^HFH+ql;D1B?Ax{r%SHGJeP7`c|7O|ZwjuPxmao1h6*TAnTsSl|Is zje;e6kp?b(gQdEk0bH1zVF3f})mh!u7rNa$-W2;w(R@?0eJN;+enP(!o$ZpspfrQ< z4v14HHI`43lYVEX4Mx=02w2|l>~~tv>nei=(|a!cB1oWh1qQdz4^4CMv+(G;)9$?` zyjsBKOS9e}G46v2stPs2dbxNPnf$%P9^CJN+C(X^+zo}yBon2|KaJS&RQ>3ImZKP3 z$_e^Sbd5;WxBx0p$)<+Ep1-j~zg;E^@slDn4Bl*bV5pudRuYUX%kPvh0)Ql8OJGr(aO6+&YTZ!^r7GgH4fNVJ)b4S3N6qGuMoz!9kPHU<>a`~kxF(&k7 z`rnOHHq^1;D^^13jwGzQmWU>-76mB(KB*^3P_+Re0)-BnaemN75tur>=XRiyzO5X$ND9)-A#m6Fm&7V9TZl+>k~tzS-Uj z+-w|oaMRNwY5-NDzfvNPxf%d*kXLyTd|CUi?RTl(V3VYx=&1?-Tj} zNyB=u8=%SkG^nwp}u)Fszbm)y`LHIrmT zsg4AyNkx&SA*m=Drlg{1oRW&7fl4Z>M~d<5O!7NhJB-|(@00g<(v;HZ{?flI#qf+f zV2cPh5cUriy(QeEm>D)0m!Nwp5;2@~QLNYJNGOH_`IsqQPNG?m*P|Rvz3yzuMl_ZMw zZt0*n##FLGlSu~dPtvsnZ%5bhUh|guWUru6Nw&z`>N6uBV7plm5t~|BGE%IV5x{M zgagY)^l~E6L-1g!h%ST)%SQA{BGHL#Slrf~VtNdib=%+Gw@tDzhT(x8SO=Mo$iY>yTWU!3o{XY%wb|`n0e?gUKjrK?%>2!lB_IVC)M@wwM(feWuu;!DM8HrfT17 zgvZQrLL*zu3XMK_Y|vmbih@RX0v#tbvc;^>Xi0^+dRP}6oLt64wG+2!(rm$ zZj&%U?5zzGOhO@;V6Je~n7G*5Buo%rYr_PSPzWZN9UL_#E`Bx%6U5NkFu^2b#N<~? z1o_Zu`*yiSMJ(JQG!6Ise?xoz)8Rr5#2ETd((jhA%~GU$!2@jSZxJtGqxAqFx};5v zgNl3?;ihT7TUJx%QKww6%3UskVQN@K1+lRxMEHeRSpU!gtL$EY28I>Q(cmBx?1}uE z0IZAjVt|!fWqx2q1+l;`^SBndfR)`#3#`N{9k4Faivd>dxA}n;6~qF&k{?*vy|lne ztkMDNBE1-3<))n8FR$`S7SQqJOfvv{wu~LYUA6QXAEU>HjftB4$ z3#`N{9k4Faivd=OK=K1CDu@MkEkCfbduf4{SfvBjMS3y7N;yh?U_}M7z^>;9R(3Bf zuoA0uz`97!2JGLIv3;$IVZ^yUVUyuigf+(7hE`s7(U+Zw^w`4km~tb{@*saZRjZQT zM5?uwH6;!4^}4tNTfMFW&sNs8hug}!wj~rX2M>dC&Qs5VXi>^CyQ1}=G4^?r^!F#R zXP~IYKF;W(6wF-3-3JfYG{YjA;0L943m&*jhIu=8$uP6ROX|Q5G8`5o$HXgw)|5ue zor6_Gm89RDjIIqi+ahbjidmQwaMKq)+T10>d$qe{c=O>UBeP5pWTJ+`i>Xh?W1kfIJ)2jwsFjK0l`$h~S#7txkf1)84k+@5K_&)9 zdyAwZ+F`I5v!RzLgzR_V%}q14tH=n*s;3^M%$DRO`qS3f;E0OgXc5~=@RjXwYOUrA za#N;EJFIX%Ha^{|2#?lY0?%1{Zcq-2hS>CD|tR#vI^O#Ee{4p#l?(&31Y@SPcCil_jzdW&7-}};r9NXzt@J`Qcrm| zeP8|LiWW<~r<)%jnq=7p)I)tqD;xj}5-$I{RfO;q+l2pLwizJc*1D;kOv)OhV>L@sV zH)M4Sc;}JjH`*nqgGR7cR02b<1ixm@S~LmsF*KBA`UVjZsNqh>15ulKkTuKd-uuI$ zV82W*7$FWJiYUaH%zUS}TOCfg^3@j=ZKmXEpr{ZtCHj|;)ZIWRlrZdU=z@REC@kO% z81PW0YVj+&_S=8gCQ{a$v{P{7O#TG|9V5+|nY2y~zJqaLedwx;@{oG)#H04kQP1;k z%|bex2_tFTcK|!1AzS9rj~gY-u4v#Ki6)(Cx-O6zl~Pc3wp5!XWWd&que#ql4}G4g z{NR*Vsr>CJuUz?Fi9&GXChAgL_DZv}3OD!SOXW=RL->z7AAP?>o=Vvf@1iKkQWw71 z-`^GAsJT7R&!nu)Br6z$5n%0y^B2|&-hU!F1!V&{go*Yot`tpMsG2Bl5BdS*@HMs- z=>GIw6dXD{(l*$XY5d*x;2}?G z*&`=Ks(4TSOGA0{P$wm8q0haiBX^$52Da8>M z2}Sf#?HoDH(yWTEPR>nHKv*W0_tXb!rb9^%dDMR*mk!e22 zi(cp2F`2VDN2Pbj6w9gZ@xLMB^W=wlOQdhFi*BBg@bXSpSVN|$J2mb1$O)MFY-<;b z?i3)w*THG{09@8LumC-Et#6alvV(rZHBxy}*t|kcibVAi*3-Q2IXn49foLN%@C}YB z5$)teb&p&}e8bxf$QG+OE6z-*ouL4>hkHXp5aDopM&4r?1vd9a1R<aHgx!ZQLW99OTiZK5`dGBJU@Yhp;>Rexy2uQJR)3uf4RUJ> zgdsaUIK~yvAU9X)xTnxHR72dALbmDa288&8Mwk*RPWOwXbPp0(&NR-H%jHX@68nFt z_Bgrf@9e+d>-$H8?Y#&708(Few*1{*|3MG($(P{XcFAWmb-%w2nO)jDzQ_>|nds-v z)IfA1zmG6ye3ekY-vgVL{k@$J;L~9Ls1E@Fgha`kBrNCsB}o1N)e1Ey~0;i7mH~ApW;ajDm2VLnxH&az2K?GJxorx>v-Z>k{YJtUJsY;UvvTO>&o>0QD zxm%Y(urvks=;it85=4_>|4qXGS?-iGs2wab&V=+;R7qCRa!9PqH3amh$SIj9U6MUN zv)JuDVxemw&o*$*DHx1W?UF)Z`W68##YAmtV&cQk()X-JFQJP)8si`+V`?t_%g8bU zTeh4-IGGHvYXh>PgVWYy6BctrM0qAtcO2J@ja1BlW6FC$2>S<0*&=U`dtx*cO| zn0x%FEkX)2_t*_vdwhw!3q}#4@jt;9aY_lpX;v1b$RCzk?4vN+X>rSmQjASQZ?oRm z*j%YEY_{(&Ev&5fx?s-GYcBNG?k~4jRyW$c=4zZx1FX-K76~taQt3GoXZ%BMI1<>O z^fU=oVPsn`0cf{lsuYmM<$)JGGNi4aEHZ6<2P!NTS>aae zF6d!jcWUx&Q!kEe!7G#hlBBI&X)e{r+S&>FXlQM1zT917wXHXE>x|@xN-fBw9aj7Z z1qBG$iplXaR@)weU52{fPLavhgvqJK3Twt{Tc#u)fMtrbmou@uu6W#ww>&wgC+90; zwQYfBBazFs(mxhwIO1%gxIJjBwncfR(H(!Rwq;7ivD!AOeNB1KioQ5j+eR^=W3}yr z-Tm$ESZyn`oTel`nbS^ftyX1%&9T}xoq6F{Z97)mwgziZ!ZJZ+W$3}nqCD7k~A)k+Q9Q{&}`~SQ`e*FYV5rd#nUp4f()1DLmJs!gb=CM@4}EXI_Zg(W6+idB zegKxZ;Ge?$hiA#O{_e9100|ing?;moM=%e2I9I0JhgtY72cN-jnL^*yo7_POih}zx zGA2?L7*9ov<#1UMAUOO~M7Iq;6`@(fPeokO@KX^KGyGIUbcCmu$a(I?0x|2vzW@_F zz94;jjE6y46deu9o+D=@3M`!1T=^G3b$cv+)wi&f+!D5u14C=(a-Qs24bIYMby%Ur z;eD7U=+_9*HG1eX4VE9hqD!zgEk;%@!!yRTge*X+^i49WD+C6&2r`byG_KP^2<#m6 zDlSt5DI;(y`-SvQWf7&*rz1x0HS%<;!cDlqb}Rln3xYtd7p zHkO{1yUkJgUAGcUv8a!Lw(N6L%cCZ9YMq~=)P9kWZ}j^61Ak}xVc>!iXIF+*=p~4i zF1^Oj-#Q1gURub3ttbeRUsHR|BHMG6nA%~Ph^sAw5uB{{^sQ5%nV)EtCTO3W6k8aI z^?E_~U4zHX+vr76n71AL#$dby_T`fr&lp-KX)qVSe7T^xFhPGUmr$LCGR z)sv<%ymqI7U`wnFP@R%t^>pr_u%R+Ndz+=Ew2Yq{*!RF?072ClJyZ=hRCNrM=!%7y zB&gmcrzKS2rl(M&2rS{^I6PR(qX%o*4ORn#CAu;kSm3HEal>FObAQm`!CDzTSSxO@ zmN8hOE5m`cYz1qjAh4Pv0c#KMSWOJptmF^^OfX!cGr>>#RES@QRqzi97UGH(u4X)3 zF$ZSF+_qLnkJoA(UdF~HIx`$zO)FljMd7tJdc4-+@G|x;(V5}!TD9V}Ruo?AqsMDK z4liT-5}g?iuQe-P>qFp$7W=xcTy8AdPb&|PV(cUv?#k==D*qB80l%>1uDvmQ?T8%- z1G7AQ?dbdt*S<1*?dZS@*WMhyc62_3YhN9{cC?#@YhTM(`|DcPSInLB=gJ`1OhYyy}O@(pNu3r09o;7X1(rqaPw_E-Pg1cl6nSrJ?^k*oDR@@9f^s+utn^CU>8H;akdnCRs0aMLNu z3f~zYvN#)j2xEHA42HH8a&SMdCWapS(XPccO+cd@8jRM5nCHnIvL(bhWp1)=xa2&a+-qr%6cdpx!}l6jp$v&jSAo`2{SGu5+Ea=a!(G#f35* z7Y|91&Ei7Kj*ExnsAh4Ye#gZ_l0>t((97fEAxWB9Tqx{u@sJ?NEG{(pxOhm8Vip%F zeq1~xMt~vG!OsQiu1Mtb*N{<#hXm`KH?BPmF}Q%`g#m`Jr7Tefd^zlW{m?t-%{=^x z_Jdyk%_A@>+wIjpdUjb$_cxZ;);isVPIq}@VWqybzR+IktSofadu#0`i%@ zSgw$}XY`(sB@P(J2At){3K;A4^5;0j`aR5;CS@IP7VH*rm2fivQvo(qk?Wph~ z!O1XjqdY}gwDu=Z?l@oT`h8UADrx8rsOO?O^^L7(h}V4J2UcIu@inrX`aO*sytqM{ zIbP(W*d+DTuPX5bwD=s^(60qF2pYHd9`*W%EXdgr1XyJIx-P(;&Xv6y(OJcOO$vrw z*c?JrpCCF#C@yP^s9<%V^a9ZtbXB@aSrR3%tV_E)?*;OljSl%n93_I`09qVXh>W?h z`H_`HpQ{Ry2SqS&lb@=C`n@EI8>6Sk%InEUf&Sb0uc4`4w^@SU!AH_dgggp|rXaQ+$l*>wD_! zb=Ouu78IDLl5QcBfZ#n_D_Bxsw2~(%xwQo>FECoj6Y=|X8x1%zpV0yeCg?DJqrX%l zPw27z34SsD97yt1mTI1wM&%01dZ6opavK~?wx+?(u2T7XmOJGyO7wS21f?8~c1S7? zv|jF-?mk~1Fg5jUpW4%s$~@QFgcb< zl?gD2Fn?Be6><^W@ut{cisl>poo~cJW7M7wSI%}xVNjYu7&1wHlAQEAJMF=MB^>nT zcJ@21=XE!N22+R1rN3xo$u_us&IOu-pM^)yGXWqklV-gkUL3l6$li!YJn8LAY#HD0 zL1^>h%~5|EvE!-w(Su0j#l*Fnpx;E-h*XUWpaMH;_g)_Iq_zj8F42EiCJTC2r5FZp z_HJlkaEftT1{q(Tk`7c&4!=Q9M^8D*L`q>7uV3pRbAgxX};?=&3@HJ)7cb>h&~Q@#yQnfbW{g& z4>nCgLJX>psGyj%q4@rD+Dc;_RJXnxrVc(;VpXQ)V$W%}QYGd0w);43yC z)E!A!Y%T5?$HmZ0Yz4_lKB?zMP_2yaDq;9$+=@I=?*JaLJ{`xYP71IvJ6fIYTDJFUtR3SQ<9&jZi6$AcedoRd$ z>$qqqd6|{yuawARt_DEn<5gbV+28NKw%?6GB~JMD%x&54UDf2ZwNY48U&WFZ>oQ(C#Zao3>=$4IAr`192Qh3lthx+gpx=|nNSj$$tRRVw(|)kk$*r! zN#r$p4wqI#qlzs|(F zv$ey>W%53GkNfGDM)#NgT`7iVWB~`^2EzWKNWP|SPL6vN%ZCW6DP>gw*Ha`%G)EaP z=hi-7<4`>1vULd9-1naM=sy@YShFvjOvbn#Grjfn+s{ZSWzJ{TX4)2uOv~dcS{Gwv3}&qBm;MH%p^CHNJhK9T{kc!$#Azi zW|CKvNY;$TXhs>Xc*jigS`x{cW6DEv@(Beu&0{8cJ&9z^`Z-8;pUKq}XEI!PkD20) zB#Jc)n}gziGAtadTnOB|hbI|n13( ziHukr(Ysi~B-M@LCY(~ok}6;wOSMyGQ4)W{Fv>I9i|9Jt7iV_C)ABB4MQBvW2?&a5 zRbqU|s50Hh=8dXGtkFxBB4-2TcZ?G>)M5k{w>XNyg2j8tRw7hsq0nH%*5iT3G%nPd z+yrR!Ie_d`l;wzFd|cG55pEXB;1C#V8{u|b2$$P7)@zSju1nB#;-wwM(fee&3#!DJK#jd1rlPH1F{S)tLQ0vj}# zjG~|sZcN7sjchS1G+Lx!g9ek41DY+{;Vs;=(n6EUYNSek(|T&lrm|w9Wezq>aQ`is z#PN4lJT@|McthbKo63rZmPpv}!2QpNhhh34d*#=cMghog8zyn#wjzkQwQPc~)A9-| z&IwoR;_Tw)h#iw`9B;}=;=n|YN)s(>^v`SsD%3Ur}dumuk1+gCeBE~{2tbgc$Rdz2x z1H+2uXs~GutmM}OU|pma1FYOC^8+g?hy_+ep2!8P>|R=6C06Nxb&*~SuyVi653Hyl z7FZFFBNwo;duf4{SfvBjMS3y7%1t{zu%d!kU`3RYT)@iir3F@El@3@J>1l!GnU$P- ztQ2D82Ub)N3#^FUk_%Yby|lnetkMDNBE1-3r3fTHu%d!kU`0fmT)@iir3F@El@3@J z>BRsm>TxG{!z}lK%cA_6!u&aE3iH&gh~P%v{CY2M^dZ!y=mC2c>ok9=J<}c{_K> zFtfr-;y@os0kjlaQyQ(uBL?j=Uio!&NyBh2zYR0MA!G<_}cp1)2istAZlWjIcaZu*ew|2#=%;En|O z&P8~jf%iWq$@eE)=OcN-A%{>XHVe-^Vk>ERdBRqXh2x1PB#%p06bar_WdWo38H{T_srUu~`~wC^{%3rp+w@3$M>_05%~m86jJ&y!1=`+ZA9`KP>_zOR0A1d)f( z-^(fE$6u>l_vVNrq)F>qI8*xr>}PgG{S>m+9Tz1gZ) zrXgmx>s+_)67|ZN?5~j6S1en|jml|{5@)D7=y)Ytl~s7jTu;J5i%oJ<4G%t6oJX@Ah``29--_3a&hXMZs04gB?HW7Q)pnGPJ41BfucL~iW14|k7t;H7r= zQMCIh)$i*eE!gQbfoLeA|-`{Qz+FjO;>y-+;!UO1-QM)0^ z_P0xyZ^7#j<`J~E?e5#M#n%tMUApvI`|!1+ozQc7NbEm&XUz&|0wp!KW|vdOt~lMp z4-R^a&4MU0Cq;KW`7f2A(d*W3wjQ;2jv#@PW!A57@zma6noR-f9aL4*g>I}zha#jwRv*QZ+E)>=Fy%`4AkG> z@%N7oVHJ9lPnT*JBRs$)nJA7uu7$bdtK^D}Hw31xvE-bMj#=9Hq-%lN*Dd6;nebjo zeUOtXx#%B3k~GNZ)$43;Zuh$WUVFC(TasLn}% zR*MR(RichGu|-AGD$&0zg;UN=Bo%{ zW>&Frkg&As?iIfpU$xS!?sOXR7P9q}&Ox)<6W z=x0*aW|B2Zti!9rQm`5Arj8&dmKQD8#E(%#a*dC}A>Dq76 zRj&*}Lb)1G-#Wz)1;_>U2PJZX*9NLLrn;xjB(HJT7kl`2iv%|BquM!gdhcjgB)f$P zIYj~C$OY>xrzBWU8se1Gze?XGr)8}$0aXLkthb({-n*m>vInmW*M2$^w7{=xGU0EI zO7D=Vd+og~kN*vIGEcs%x5x=5AgPOP8h6SHYsl2;lr;hrK>e%$W`A*qgvT;)zBe7X1NO=Ro4- zUVrGWf9C`_!K?Mo-i2JEFhvG~cNi`7v1n_-SkNcLk5PJckr@W9{yG;Lz+c_Pz`Zc3fZQw8xTBBlqsR&bic?Xd*{eZ<4n0+zEmo)|CefyldB-1 zejj8!M}zIX2mavw?ZeKNzuW6S0BJ19#~y>^+H9uo_qQQa709!N>}&u>-kBPRPULr` z{QW$6T_`BwZ@QzEsWY`@9sej4_pKA$W(;h2D3^Ep`+GYd1k!P#bB1n^3EoGz4jlG2 z;jmyaTz7VQJKKX@Ab|Zib$`G6fp_7}^W+>KP@jGGn;R=D>_^c3ckz$xJNt3DcAv}$ zVn;~aRdPXf7_X=~fG;&@jA6G2wyQhRn0g~rmjGYQm!#6Ls=HW)~PkcBnI`9qcri3F)h-65iPWWX2K<|yW87Q4MiEK1kyH{JryIR(R2s$EjcX8INZjpambYGUHU&(im-%B0Z69*uEU zbybyq8Cip2Yrb;`CzAorgw7XG_1J9B+z?UT+|-vATr)PzGiz?@QB7U*%c#%T6mM3J zvt`CO#`q&;1y`7f+iqBg_a*Xf)R>JY8cWIZ#PU0C%dXXKce&H-bQT(G&CP|C_G)io zy|cc#u-1-oMjiR&y;pqFM#pKb0o&%29$md<={z=^P8>DgF(n^#BB#E zm^@N*RO5LO>00Y0IACwbR4JS@;?$OcM+QCUWMO(x;OaY2VX4Rpw_0~WQ~SEpo+-33 z^Qjj{wlF#&fAI?W^%Er7fNykvE{2M570~PJ@DsIn*6mB3C zhOVhBmO|63zOk+{&2R8sYKnvJOjA6Al(eFFkLN1~u_jYH&ys2V-DefB=D5oP|K=e@ zvd{Q}xiVEMZcujzpF!PCq3`NVF4u;l03JGnR0YOU5rrUJR=CiIp9=r)@KfQu9DXW1 zZ^KW8+i3Wy@Ogx%m&kb`^=$|ts1Go~;|o$GY&;CgBL5*MdybruC@_bcxiVNO9gjNx z7G|!NFmoj>8)%|J9(3$ywFUm-@S^Vt`ZYpyjUM_;gXKrB=n||=%h~MWS8Nw1`X-sx z6#_e31Q|zU8rNwd1a=O36_+W3lo9A6ej&XsBBEmYbVN2%BTu)OH6M>=68|ZxUAU`O z8)!bcN8|7#RbnYXpvlItM4S6ePhF1H{I!sXb?r?K$e|=3r=XwKjFt1eg$>6k8bbhQ6Tt zuEFD65xpo1^R|QE7>q2F(xk>ShSo_M%mujh7Bm+o=&uEH0i*`TTtKY^U8d&*A;AOP z_0L5neuFG=4~J^{zu_rSp2cI`}eBPLv zJlGZy58%45$D7lZ0jg8d15JlZ4&QI-+1pI`WEnp9X(Sx<@2CIR=5?vV%EYNUD+%QUC|db1cGpTrAy*qUEx z{|hvTa5PYyIRrSph0GzALgo@UZBEA+vnU44GYRc8YA%lx{Z)kZzag&zB7GcEb>Fhb$HJ z8aBk;yd%N9(97E;xeuxPc-WK?+F?vQpUw_LQ>RHtZFJBI+vuP%XB>+=x{asC%O= z!R|GG(^T1dcy?(%g7vuU=0_7#!;DW_|&Sq3c{#?zsi$W^tiI$Hha! zW3#wWuH)h%@u^u{Xx(w~kO0vvF4Xb3cu0U|78iPYTs$N{GK&iZJ}w>-pP0pkW*-+1 zNe^I%&T91pw|sP|$>*;jTM7@!(>Y6AOCMkCBV&A3-@Jc+b8WT0P;al@Us$<+zp>C> z+GxP1I+$OrcbnaAGtT(xB{06aOD^9B8*WQntqpGt2T}0-pqwMf*4p~Uh7X21&9Oxn z1daIDYSHz7Q1Txq$!GA?;bAvYhb3BBUTGSpUW?=!3oaBBVQ2<4>e_(OtCJi2g?frt zgv}qsY7(zBVqQ5N=C2%g8PfJg zGn;Nz=r}xv3Y8bqw!6IF|0RaU01Gf~>9Gjx=)+!jG4SZfX-!h412TJ2C8EXvX2k=2<0>h}eQdKV~^8qG8mMTB1kf zlJ#(jP?T^Q#?T~AN0TNWNg1%1~(L0FQ^xO5JcS)+jlU{soF>MlW1&Y zs^cl^9Tz=BLaXznd~bUX*sDt8BskkLf7|mAApx99J;0`MLOi-ou7IO6xU52qx<@?- zaI)8a0RFh!UE$oi8Nu})dDG@+GF*7&6mGT)WE!*+o7)e3h*NmD?_KvEBd16iyh>)l zc94|ehBG_SnEc6!_0s=Zny5WRN}JyNo7K4Ra4Sn|4G8$C=L43c&4tEMk4nyff>rvz zm5hjj7r$cJ93Y&ab#bZZ;R!lKm_ZzMFg~SG_P6}du1Ab8+nMr&I2TP`2ojZJTgocfYZ_p9-@%cJ~wU z*O>@3RREu{yPrRH_hXL5VIikdgN1w}$L@X^z#6;zWdLjJ?w62@aqRA=2U9 z2(PibU$*%*cK1ufYwYgF(oLx-vWjDq|rV^GHHMfe}ch&O24HN7|6#ZwgL|UdZIL0o;p#>Xn6k`Gra#_-h7dK zE_tE=uLSnNhe!QA@Ilz?a<2(CmG%dN-oS6J)i>zA)vc!L7_mV2Kyw=Dm%z+->dyY_ z2Oj@R=f2`-X$ZUf!cv2*gspKuMxJn`24RWL`2QM>>~nfF2_gQ!Y4U$g7BEd3&8ByT zoZ@LFSSkdtRX@%BkeiKVyq|~!9r#zaYpA_Q?>6e)6?3<_Mo1SjNcH??V`U?<-OTA< zn{1nk+i$M(dKMis>)WOOpS^dFjqFPA#CTa`vv|7M&AueN*{2_3Q`34-!(l!7D9=Oc zo{_AP)Gakz(u_Rnt76q}V4wL4W}H$Jl)5b85;@5Z(DXLq6aa=*?WKQ>BV>@!cr&SJGY;EV8jIu4?%76*k%SIm6qU;G>b zb@~;xV|IJc6YHI($ELqZv8-)e6%HL14o2v}p+mlZO3XvRdm7;2Bs3sQ&j7fmBIr!s z>C~CYVz9flLGK7JCnttFb6j@|1Wpn_0T|yTI!YjaAb;>wP(uP@DHFAuz-5>i;sQJ2 zwbD-`CdM|ps1bqcw5}<@C34_V&RJaC>(fA*Gg4D&Re#{1DJ>%tH}~hX7cIchK6J7*s~ZOlI&L$yQ$o!qmN7szFPJseAcz z72`NW-78{9n+|d*|8l$Oe+VNhCs&??iWTDBJKf#swA%M6P9-2JbnjErXCIw-t=@a> zaM$I3aR_2i0d$D)c|8*`{XU0gYp%YK=JFpo*h6`i4fF(&5kq9$iKEa;o`H{Y^=8u@ zf1%lYtAwXnW;Fn1XdX7ep$fYqdl@FBm^_<1XySUS-i2IT@dZyn#>D%XlgbD7rO8u0 z2EVO8Oit4_&1)3&S1FkHA`N^Ky=t&{7&s`*X*GsH!kevod;2YrF#m!>gix}43A_IS zehZM6S*p+hE$J2qNOu$E_F(MfU;yE%AO{L@&H+aUNc1M?R_pB@mwtvSUm762H__JlQcIK5P&Fl!(Jk0+ZnZ`fRKTg zH6(=c077^Logj6E;xe&L(y|X&$Zs-5Bh_MjMXF#d8(Pd@d@KyeN$0@?Q5MVG?)wGa0^p65}?03H0Nu$YE2Fl|ep3dPYY9ZXI5S~#9s*dW?5>{DMv zV+z5$HG@75rz61SYDGE_O`FJjI`$-R2N}Q*QYG(-gT|!vkNm+n8mA}??rEweCTJpW z@3eM(e35}Ss2DLpTi~l!WCuWfVpgnEM6~ja`A{R*c0d`j3+UG5pbwgk9sajrg#lbs zQiwZJ@H{=40aKquHzj;^pGD3c_Y9gUoy_HObB;s)cL2cb0^ARM01#D&-B$aa*ZrW? zYwUP?e&-$>g`RQX+&1Ttp1R*@-D|bsHi=ao;H>6PE`jQV{}%S}Z=hS^W+dU#N^Vgk z0xA8)^$x&<`Ma&|9#nul9NXG&K6GbppA}F+`UJIBULAVR}R~%GOI>TEf=(>&j zD=df{;MnhDon!ynL5uy%;PQDgC=_N`IbvLt@kakdEc0fJ-YZIcvSd8XyXcfU*V5C& z4w}=?djNSp(1EfqKT70e$#Wvxj~j%wGsX{f(8H_9Vt6OZ-y98dq?AG@%jfzxajgdz ze)}Dm4X+r5FZxR%eC`zWQI2$>bEk+gtaIl=%(+uxO-<@?o?c-LY|bYGr7iRoMQ5B! z(sm?$xfQiRb%X%sj+?6Fqpu{RDpA@vII_&%iCN2#&Tt~y1HE7Dm9l2>swp(QgeN~QmDkS767dB655cj1$AI`Y_%WwogM+`IfSDcoN{C(sZ~F~o-U!MN ztwkAg62x9c&&t@Mz%1Rp{U))Nq*b6-3{a34(3fO%Sr49eTum5%kv$3Co8yvToJRH} z%KhcZc#iBzRQe~uxR2~fRQo5v_>k;LtoBcWaU|K3SnHnz<4v+BvEDxk=3D_uXmb<; zPUH>BdyoiIKownmE;%5mi==ajPRF9EFBo#+Jx|M}s6sdDGa-*hvr+nY{liulP#y^e zD41XYs`^8496pVZ_(jIB_8Vqz?Ki>lRpZ0Ft^KZ2LKo1?&IH4UPU_ynE}^y+P`OxL zMh4ihHcjcPn|Xazk~yT>%D-pIMcvbDxeRi>Va!3@+iN)ta(8gcR0q{3QOiL&mRK~o z`0qRDs%+b@QJ%f868}6l4pua)qqGaUY^WI12=GhDffdHz5-W@!I{2SE=*hSOar$A& z?tMwdCf`Wcl&2HCB%fL;7bq!YX%nB}eiY_@zryE3Azwe-hsIYh`l88&eu#n{p z#HCi}r~-q#s?)NlaI1OK9V4Gu>2KsYorr_X@aN?_(caE$7{n$b)VOb)rKivdud!PP z{Ps%-fUtM>8@1y4jgk3L8T01Gd<^#=WL2Np^J55P*p zf;I_Yz)PffTRvQpq}J>9)gk|qvK>!U4j=9`hNWHpW#t%=D03#L-~!e@J0((YJNO^x z&~kjI2sMK{y%`F@va1-YK#XLAj*h>WDmT{X^6gr>x1p)zyI)TCE|;`?e^$OvhJUSl zxCiFGW(zF)j2tm2wrj&1NXb&3d)>)*{LX&&RX76U_=B+=m-CU!L~xj(*ft-JD~G+6 z8-$Sf_RX7g7ajb%&X3OSw0JH;@BqhdEXRFtgy*|5tk|_b%MASC+XCpy)aNSAF z6=Dwm589dGSyu+^B8pco5qvG)bpH-eWhtdWUbs)%9D!W#U z)eT03tVrTh$#2@~q3%$HJY_*tC9O75Fs+*`NOFn(_qM~FKK$ijF0D)(s!WyPs>oEW z^iiY==4rCQ0|g`N8ufNyxo$aV>riGZN;%Iy{sTADuUbVfyeM(9J0mfnH>q(a@|c5@aKW*5$VSgE>pEhiH z{eLIW=WL5AtXObKcx!vR>-XGK>Mv|vgJ6ZFjsPfTfB)32P>#N}K~9CIyk%=N@EH)Kq?Nis^QSV$gKxlZVps?tpTqAKmwFRIc| z{i4d2BKvis-ko0RMHDex=)D?S#*gh2|L;!Bo{=b6;RHhdU`4;y=Nv^WtKn)&Ii4zd zM16dz!@0Ik$2gq25)NC7g3W&Jc@O_z+6h)K3#}&ExIW_g7W)lr*|CqM#;_DV;u@Fw z)mVt@N)l>`oqWm8<=# ztjzl6yMZHC8O~OZxXP>jsw_7e$J5Gi#Cyb5Uh7w7xnmln%KcX;IB7oODzEpevfTPv zs%&4A7ZR+=aNvE!HQwk~W4VPLpvM1?X5nDmg}}M{=v7v31@ufz+pc0k(OCA|j2};> zhqCudSLtHkN*`hOwSW_}^iZ~6=_+07Tj?Wgz!Ft@C_Aupl`i+K^bz)8i7GvmO<1~0 zSNc}E?=~!M>Q2zThq4VzSLtfsO84D}#Z`Jwy%+w5nifN6e~h4L0W+zn!HN95>ZW@LZ~KCs&fEute7{76xAEO{#xX8LiuOf z3L45WA{M)Xjfh3HLse`gLWvGlG`iI;tj4kK$iol~&oQlNj&y@t72 z2!q2%AsU`zTG7bW!Xz5C8p9F|a~~PHXeg?Sc|h?T(~3r}DJIdV)krOx1@m0P++rT3 zXn2lkMI%=plW5dx3`;c3-RDt?hUb`8G*VSy5{+7oVTp#hF+EDr@Ep^MMyeD{qEV|c zfM|A1i#K!6N-CN}7R3wrpBh(fo>f|zNS%X8CTjT&GKt&Y8RfB&YKM0udGM^#%0p@- zO!836pHd#0=>sp7S1cTdKz>g%i3{fyk%;}4jqU4{%R^-g7*5u=fwtNL&R%jes{1IAC6$T3S@A>vIJrU%Mfh_5-cxXQo(XrB@?Wz>cbH% z*P{$BSe8JnU>VNOK!WAPODb3{t7L+;RXv?x|9wuiuN6Q=bl1mhGTc0~#sC%y!#;)= zB2r>QR9$Z*w>*l!9Zx3E-b9i$#nm+p@#)%j2c~pw3!W*iZ4NiZwM|P3ERNa?!Z}Zb z1&~m}GCMEtL1XOmM)7|$s(J>pWX$cX9F&5dD|Yum0O@9`iY9mu+h%f3sx6dLOJmWLqwXEDj4Z4lBm)agT}O&6KVp$55?e z;#Nv+N$%i|nUI#qnBu^$1WOB>R)U{oh7)7ePK_DDFK5^hJq4j##SBzda0Ly3Hg|Y; zB6CiRvG732RmBig6pKo(mYvQ3`ED!us>p&$UyW=Kg6=Q#>$`&PDEX-Pf{`AnEvu<= z;%iP*$*HPfN-8<2GZG&mhMeMtoQ5y{X#<(7pz2ix>s)HQhlh3k|Bj+5I3oeMjUYYw zUq{i8M@?KVb2?idfy@DwgDK|OaLfhZd@%&3!`(50_bd>=D2miGBwQc5x&Z%Yv?z5CZxwyTxwY|1lTrSqvww5bfTczcC zVWR}!ip^4Wy;*HGt8t+3F9GQL^Js2+zw-fMW!^q)Hz@jYyT0eUPrKJVPdvJJaL7oT zfdgpA+XfaL;yLN97uG5XVdF35FT3-|0%=mal)n`F^kVE&z!$WpypX>@5C|i?13}?C zT3g7^_lDbcc zYFG19?qw7+{a&jT^QQo_+m28drT&3ePNsha#lB*gLayW|U5u$m-1t=Jt&tZvE}?|n&?GXN$iau+SIW{XBpV-lh+P?GKu~uC)mJX zx6t&~i)uw%2t3f zqb`s{Dm2s~^c1r61SMn%!ZtFMqU4gkTZf?8+6J6Z% zOZgMAb55!WY#Ao77O>!}LUxEqDzWaC`=Brn2dlOVbv%swg1dH7RztP%m0fI2y zdcl5%^@Hd{Qu{hgXD?b){JbT_S=YdwiT-#A{l&PL>jLIOIghaU;<23TE}%J9Ivr-T z02AHYXp=o+lydb>r~dFkspErMn1ulUWAC0190&d$520}mVRVsAt&FtY+23QoJTe;w zdG6JFgbkRnYQiJOy@nVeC~%O|M@5)Z8;6|^Y}eIj^Jtb8Z43Ce30E)qOA;{5W;r~I zrdez;!@;^>3N0B!%ky)F6bLOtG+iKPz`?SjNfeTP6TLBXc|zHO{4{AoL7qfUu>7pD zgh=N&87!5ckR*hE*FkIaZBB;y0%`rXO1Gi)c?e;me97;<46U#8O7^#2OXTGAAK@Q$ ziu=7$^lajysPw_Sboe;Bv1vJcK5;2^fyvb=;ejD%B;mxM%s<77@LIk18ZeKaV1IET zalvY2FvDFGpA@t%O=E7M@UWyOUB$EMj zaY0FbPL)*+Kwz7&C4jo~HA56vJ{ctA12o05xehZF;Dx&r{0rvcP#_9lW}MIl$?w_N(1LSAyovu0qmF_bmo z0A0Xad^@nhbAWCi@YS30C`x%G&SO_}YFI=W**qIjGEaI-X&7dM@Z7v3&IE+P-W?;K z*i@52SbFY>o<+6;MOGJf-Vk-6HQ^Hp$y&diNG!5#kr7$y0D=NWB-mo`&H$Oyp?HS% za#|Mk0OBMk*@!fMDZcp!X^^+YY)|9?m0?Z73s37eVBzlzArT3bj6JJF+krM(~dTN1nCu9mH#e3{)hz`t2R4kUBHl=z z<_fJyFu`D@%av%W*=A`I`*!UtoUEqupG*K}(@zLz(~lf{OU!VjBa~Ji9#LivOqqQ0 zrwNLYV^9k4`JU$+R8h=tc9nouk9Ik{%a1F8Y6Ql%6xsf8K~EximZxi(hXbeHmHB_S3?>Y`{pwoc~)e50v7CSHd)p<@odtVi2@A>QJIJZM? z@stA?QRLq!{Ezq0WRQvK9LHsY&j(Lw5h7a|yFHhQtO>u4PD{(Fm80ttMgZ6a$rN4X zJBBKdCdKFFbCP4KDnzey@oh9M6ZDEhsYDl@Qqwe4qk%7|F@{l*H_(}3D|SHFJFcm8 z=0iCI!pA4rz!JMmXX%R(gODZk3`Gm09;or(qKcVKbe84~EyVsFy|X!lHf(~YDwI|@ zl8dZkWWq>NTj~oIprkFBzC~;MO@tIaKTakYW3-_eHap-%!FRZd%SS3|htSCnl7HDm`+_2KK%0uc|mE#vy#V)L}DlAr%qY0~Q6jo(O z!m1vNu-fX3RaJ#G&36caOjL2P!UT`xtP(#E7w|90fmkt$s~Rt^m<2PhPg|?UFR#@& zd1(_DE6ma4RW-_Mby)IRJAQes#mP&XyI5h4Ca+bayw-*#ul3`X*Ls}1wCRf#=4kR- zGs& zAUDcDr7AK|sf!F$DkB4x+ME+|r7aamO`OoD<%T9^!*LCR;x;e%;h=>Mo9*`Pn>X=P za0sK5=ZsW=1v%k48K_)Z;j^M(K}=bccOxh6SJ2C5KePoKk-WwI4NSE0Hk#3HZdRxj z>+sLv9-1@cLyS>w^udM?o%qcJn5YRq6*ET|6t8wX8kRd7#j_Bvo(+hjC0XGKp4 z8WCU8;xp08r&kP5)m~wIE=PmK-#e*M5L_=$cXv9i_I*mw4W@_P`#{p@-~%+V`Tn8r zf7=J+9rgv!gzDz)$cyUa_r6PW=v0@bNKErUKZkPIA!@cso)Pp1d!CJ9=BaDOOGD6% zI~a^URn>S&Q;na~T3hx4RaH)l5j>J}fX!XaOfO*E$gZ42(+vtRKfZ@zH9WWx71BmAZe$Bdn6=p^%eUMb>dWpKw#UDNsZ`+PPp{mqV@)OG`-f>oB%CZl5yS0-BIl&5ZaBHB$-Ijf+fkk z%dgyOx4=Q_mBcQ^aS75J)&?Z#3^T!<+9tSE-LiTVo4Q_nDFGp&Yw#moU9WC@CVZ>X z`;n5WrS&6?d&lNS8uyOPk2LO`%#SqlBds6!;3|wp!3r#;BYfsZ${gU)e~r<<8bx1L zZJ{VW$!`52IJY(cEyM%cSa6B#9d@Yaa$(w?-jNM02~4KN-5@*$-q7VpGHV* zaXgqez-H=W2b2`xS4ckI;)T3*@Ub&T0zjK&4ljkm@ylqO1H{ok#FB`tAad_*zw*u1 zPQfyr&;Kp4p8uhP-yM!FAms^tSSkSL_*DRZ)kg(znl?eOas3p||5k%_>v12f+^9v% z$^m#U`^}Z0IV=YU-D)rDF}HxhKuk^19eW*8#+EZ8KCi}Iks3hKfi=6CTr7O<>*yp) zh^mu>Rw+RpEJxiDFzXnnDY|1Xwsv>%oLWxV#Is7SL8u&jiO-FokDw_L8Qr_7vbV9V1p9P!4Es0;d;6{c(}x7@GNO)Kyj0iS^3~A0qSO5~)$2)i9<` zY<1M@SA8L!v4bJR`I$s=`?YDk`?KD0ipT z2CMUY>4Zc7EuBSUcc4eRlW3we&VnwZ^Kcf}-Sq)p<^fSpvtKLc>9-s-MqYL4C!W1)m>lpq8XDO~itum;U0Ot!ym~`H@dGLKI{UkP z>H<}hvosrNMQB@1kyuthB;^%d3J05wumbNH^i=JNuD}baRC3G#r?h0AtW7g`=?k`r z&^$i|@xx+sS^RKt?OUJ|`Ua{*Krgvs#bz;a&w4sN_&2k4@Wc zueu&O8(}nY*uLN1|DYW`23k2|BevGJvpQ`c)`>iYTHFQTvZ z?*`vy0KO}xjmd(g#g0795!Y*VmYct8$>|6)*U+vVD&bk{Np#hA?Z9zaYa^vn z`aXR;%I4(GXw_mA$R*yqwl%dQSP4#xfkqbq+O}4w< zWzJ}zV*n(t%oz>9;>i{>(P!q2<`W$nxNHY|HaXbh9Xm2-GzA;5pQtitw84a>7_(>2 zXkE6cIE7H)YMsMM&r)pj6=7x0Xi{973yI=vpolATMpL?%CNIT(LXlVIjAoNp=8Ohs z#Q23#>tC5On&6dL00zX&8BM)3HA_ZW@WKvVnKRna0Z=k$GCQz9(T9iubv2jM*Xs-Kfj0*b{sJn(06M}W7csz^rGWef-vQ*8lwG_@-S!!Gmwde_5X z8Co)imW`nmV`$YFx@rtvGls50`#6IexH=E$IhsY5!s+$JWLA-Li2 zW%ihalg*%oCG$KORdTd_)iNi87S_B}s8g#dACu)$1})s~9_)9835D=Gv-@0T(89pW zbnGb18MLs_8_r&Lx!!PeF>i*f&Y*=;2rGjYju%!2Eu3zK*W{H!3+F%&#cw>!GnXc> z3|csyyfSFvh)13@D^PHYGic#1kVTzfM%0=mieil!wztTjg*mt_4ol{7ffoMt94I!q z-q9;$;KFcf`xNXs@L!togB6R)LTUuz8QHFe8_x_wSl45ittuV?hVY*_pyg(u!R&e( zhT&t0BAkH((;J+x!VM1SE8vDgXm%1H3Yqj|4*pIYIPiR^Qv_pJQ+Gf9@WAgGoy4uR zla5B2O~zvf5d3uLs?oK;=Py2?5Ehu$p~WY)>pIs+lSg1g|XIJH}os&}Fq$9wH`!CJI?NxHhWZMS!3Qn#c`riBkXxJvPQy&F%6w*u^xL%Z&;cUu`st z%V3RFTyAWwRqGpDtA3%`7YTe5o!bIXKmr5qZ6CG)V{5=q_FW7jC1I zUN4u{z4a;>d}Z+B{_fAJMc2RhIRrZ)eSr5Gf!aI#Q*nylkLrIh13n6J($ zB)7}<`{fMNS12wMdQ$u}V)|~Qi+B8Xcfa#Or&E9UR%ylG_4fcC&Z7q1sgAN2#dVB( zeHt#3M{4LlihmCN0pKBP8M$ZK&sSUB9%w_!rW=1wi=h`K*qYhkfvsdUGiXXws@Z#Z z;8QpvV8@^}^ms_Fb{5J!mH%V{P_BPMP_BRE;NNgiRVmV&K$Hw?>xwM+i=F*F_De6U zkehw*CA#_};>!`T@*g|s>4XJwXHCTmQ;vN(VQPA{f;Kn?S%*ov0BM9uyT2$CjouL; ze?717_IUI(tZslkL)gO3;|# zD<@Z;go+yCxn`yp^U;ad>b=(vcU}G$ht6yAWQ@6mxXk^T$o<_pG+T3Ze9&C}BZ38= zXW2kc5E-%K?M@uJRPqdbl&d$J?)VGM=36B^%`&S2sBH7FK~R`nk-ZEXq#k$B#PwFa z3%R)B3!Z?CYu^?~PlKfT20vi0FiY%V=r{k+LE|(vcYO0;x7CAYC4XcNvhPjv8U-P; z3hzZ4_$GSQ;CwW2P}D*eB)r+Wx3}K{3G*-bG%dTKd>J)H!D=FQR@!!P2-}_ovhBHxE`{@JAMSZP&!b`% zIQUOtF%4y4+Lk^QilbLL82b2HIG$SAAlfnPonAy^gz8Ofl?J?c8i8~2{0BR&U4LZ{ z_=)#G(ygA~sS_1aI$6h_1nwZ0WwN^uL^}<}>~rLz|6m-A(`4P#6#aMC?{?oNb-@>- zxdm;3ud;{H`pPI%M6~kfe5jFYJK(;x3s(gML>e?5JN$3Mc22=-N(ymD3ZADYGhkkD z(M<`%aRktHz*;?nrb;Jsx!j!NkpCSZV7LJ6M;`#h{$aP(zUOs6X!RO9-k#sN2S*{m zYQKo)JknG5JFR;y!j;4*C;$Y1{^U|4vdX@JZZQ~j_&lp`r--YGA&B4@O9mzSmEnX7 z&x-z7{G^O<9H5Bb-a)+sGzb1}tGfpk7=a3mZS6N7x-+-WqBFEXJ@VVDtgjWwLoj$~ zst?I;c(}T;w1uX0zKvt3+yyiv3apN&(fI}ZD-J3so#Cw#bY1Oy$$lT}9Q)S}TI^p2 zm(P>rv`!0!Pg6mQaZ$z_{S&dwn@i6{`q%|@-a=5lWIWA-f?AzxDLL^@h~*hB=b;BR z?u)6$zDR`*lurn~#<}aW+(6#ZbQ_VACC`a$KW+?%vf7%(@J^P$IT|)NDTPj!&-HKO zT8}D|U^Wb$JGn?ie<_5|oq@>u#u(PQb0Oy3sowo1bV14Kc0*T1rkGAM=0G15YR^cBGBH<1H`4)#hL2fWRR&pq!d zU=u2bka9&>)HlGXr1K3Wd6mvbpAaij_K6FwrmPaf=qXnRNIApbp!cA#@b~-=dyg^x z#(JsI+}>_%EN?Z+Wx(H9TVLLSyWi#2LZh&{-mH|?SGVKvH-MZdyRB{i?B3RXCy(H! zp^QSEtz2X0<_0qH}Ol&f#=S_S>1&hI09tS#5yb?9wq=ZLAk7YgVfbc(|Hb zZH(Z|YJ+>INc5r1YJ;*^z#d132}xy3kXda!sP7*7Zf3P{>N?6h?#R;E$Vg_jLHXG& z+$NdTh9_Cgj*czyv*@^=S#A7mv!;CSdGzPt;i+0}+&eh*sO1J&Ywvj5VDC&|9X!Bf zSbMyk?H?Jz#-0lQF?pv`XYlI8{Qh`uZcl$L0QwpN%-0B*V<4HLMqwd53C_#sB*#E! zk>v8(312=h;M-_gCI~k*NB89$!QB`p4pxnZ224hCJ!(acLA(9k?AMpj zGbYXrQ&`l?a5Df+k{*%bK5anP9tVT5@dQU9}|!xVKfkzLMS+XY}5VO%5D zZ_(O*6QOVWo&B!2+qxfkFkOsDG~?l5tv8G@`@NtqfR%;T7r+XibPd9@6pT@P2FzxG zfGP-(UE!;dW-6fqg91+e8SWB!{0+q1CPb$nrl-aLmi>t?{e@*e-nAO_hdhOG61)9) zM)VwrYJx)vyW$HD+qvV@)CtAVes;mW2UrG*s4^Z7!uWHVd&V(r6sdU%35%h(_3yk# z=vNsJhq$XSL;q`oFyrBf7na78P*sf}IXbxlv!TpcAs9iqD#XuO3*cTf$>%JuVw|c51q0 zs&e&tTzNRYpHmRC^RSPHgBZb(;tOc7r^l{XQkBIz2$Sx#GM?1g%PRFRi2HzpFS}?` zclV_^bgIizB>E;m7(zMhjNlHyq4D5iOrC)9KfFA3&G>LwJbo>X#p9gT7lKyQROOF9 zhsRwbmV>2k>01l%sL;#Yff&@*0d9mgamocjW?*y4GooF*qhzAuIZX3j7&MMdKWP^y z`>>z_Dx$+<$KV0<6v9;-Bs~)pT24UP&HkszJ@K<>AMjF!>fm>NhEW}o+Xt*xR|}ba z0EJh|>;no;DrmLLK7czyxI$k(d3}Nc**gx->;tqr;pvXwIP7!?Q2;aDlAEKACTDx z04-T?xRCoD`&y~rf&(Uo*zAOF!7GE9j_RqG?WtFfp1NW`7B2hh(NoXt1Hh36z@`E_ z0_93Q2vcsVVidbdZ7?F!DU`cLjI$%I$q-9{{wjjASA#0OMX; zt5laa{APW*vgX&9*H??%%a!JKp}1XI-Kf;p;#eOt`+%Qq{D9xV=wFQ@LNUQBjQq`A=C71u zUYUek#(|6*XnId8CQ4<_Cjo}N_2vrsc#9YE*1^Zl90`Ey8*_Lm6pmjGVX|`ofg~sC zT*zGSUHuIJ7U!k z`m%AQOH(hF8-AbHE4(;b>SbNB0k|{VRu<$AXeVhKt=e6)I)w`oty5f1_z!aE%!n+5>DQ; zz@iTN2p42i8fQV5(Rm<423p%jul2z9_Ui5WJ>d9hHGzj0jVx_PirA zNz)Y5#MGMr)O8u10tcn-);;f_v)|iyFS{4f1j+%1*)-sapd1`VrbkMnUmsa_e$^RS zdYZ6TE#6*0S5$Z_U}9}I>zyXUxFTdkxTW+5)mDB!)Q++V#_!(TzjZLctn^zB8Y8c| z^b^nCHB1ip91V?ZBSm<)gDx$iOJ2Rv^tKP%(1iPKXw7!7voGkBDb@09q!poUHOu<4 zVtg;%+EN(37toYP5wL{1<_UBexO79V5945JlVdc_r6uzmZJM}CU$D)8=4mmAAC{KO z;)jE4o4|{83nld5pp`%i7CNlF%Qy1qlS%YPFdr~xXlNBi{_I}jZ08<(pwf}g=iEhf zg@v=vC_^~JnL}sU(}Uf5!{6E8ZTcMuK96Q1!CrBG&itS>@0cIJAxD;x@R6<;JCOkXL+^k#TWUq{cU=3pP4cWUbYSMtckXg(s86KIUayON*m z^E}_@`9$pVOZgM-JhC)OZ5i0po<&JF{}6pFP4r7}YPaWoDL)$X^mR~4ylN{qDp_VD z?6MxZ5}inDUx!KWMQe(mx1>1hx~ylZ6Bqgqub}UbqideGz2EtOEU*?J$+gu=Rnz!T zI#7WA+M#ry;#Y(I0&YO2V1scziz2q)5SmA0TdjH*#;`{EsVQw>CeePwETR1-Gk~h( zsle~EAun49f(#nO%evEUvlqlKQdE*a@SA5C{3aPrDX5Vwx=&FZi3ckDo<^tSZ2*Cy*w+1HZF3pTi1sV(`5IZ$jB&Mh1s{ujti3Y;_?j2v5FV6bWddC1YsW z7+NugR*j*n#?Uom=(-4n1&#s2m4IxL1H(ZzW`HT4{!{~lxJ51Q8FZ3Mfyv;J;e%dR zf!GSF^)TNT^yhX2oyf33Ki{{4e&$hpi%mwh6^jjGB za7;RL0LPseZ7K9{Ibi0JMJjB6=uMnId?41TP1Vs zVLTdUj{*&zF>~w@vvvYuWx$^)gq1n=#0x8P>`CV}rpYUF>`CXSk~#LIlUL^06VXfx z7WNGIGZDOH=GbF{MEwjn_WXJd6q!PMSWph08T2Q3B*EQem$3MHMR`Z61S)AfF+6wmjJ26NWhi3@cH zOnbr?>SnL149Al$R$Z_cnK2k*fdRU-gsUjV2h*KHu!7i=T=i1)mTU9Hr$cv+u4SoP zd_ut$H!VnuPipset`V(@SS|&(s|7cL3k(>&kdcg1bfd7yi%%&LO78p(j-XuwUjx4b zmT+CL=Jjbp!M$%ZCqN)xyr$pXx@LH7Qf7I@>zWpB%^>UXMnhe65og(RgKM`SY=?)z zmHy(2xXD#+XWbYEIQfEd^($Ubz{Y|}Z}GXvrLa3sud(E1mW=#t=w2V9>MN{_x*WYT zcNH2MhSFt;&Mwo{=plHhMJ;u9%dSReTuzqM+4Z{`ouL(3qO*H>HF}6bM=uP!uveoq zIvY#s>?U80zDDuOEYaB&zZ!i#6df#?q^$xmb#TuIze7RW5e ztBrbbxw2hcTduE{{pI?`TBBSq7whYdzMgrT=p5m0rPersacH}~=ery3nram5c|kmz z)lOxt;DPDRDqQuRy0;(12OHsMMwX@#BFhDA{C;u2LqcjVfECr3P@JVSx?(XeI^j}_ zp4uB=S@jyS8^$w9q9sc*t{X8Is=WjUg&Q$R=D?ZiBPY(V^E2b__bo8(`Ua{*HcE?V zpU^-D-$qxQ)I3=?p zyjJhMcDU=FV1IETalvY2FoR-`PYPO>@)V+vXCgD{IW${yb3|4zNV zXchV=D_leh-k*w#W#E9c0930zMQWb2=26htdiy zi|TEXLemy$ez*jT>tcg6$lGGJCrtsBVNJpdPwO{e;qMC}5ebxxJ&R#5vrW+o%YVQY z=^)eU^mYdpBEIz?8;woA(P*`s{)b1L$wu&x9kj%|lDfIEWU6i{csck$)5cU0ZzNB1 zg;pe(V6f8VO0?B%v$Sd9ubqX1+*JOP2|&R32|>X4k%MoE8IE*>(#pdl%FF@(FFyIx z1jWcPc!2R`mFF8&Q4ChQ|Cd(e@(_v~%qpZTWl?1Ap+?)vQ_o~#rJ}J{9;PVyt73~I zoXZsDu`(XZ>Ugxv;az^%2>jq+YzMCNxQ{nve=*;TB(HjXE5&6Ukza_~i~BH=p6WGR zOFkyJz`Y%CN!|rbv#83P&B+g7Ququc5WN(MF1_Oo(aVwO@{->Wy%LEoum26vtC8r^ zFd&8Z=HgfTo^sv(M~cPI3s+(7#N(QeK{B>>#22x(Kv4PrzH*dT*miX1mTT{@}Wsb6hA}Di|WmapMqwHwGq`Vpdz^GGSqYma@ z1qXB3P$T&Z2RHVb!BFNXn>orJ?lB*FFy9-)-$aPvvF*0*`{Fzucc-hWR2{(l<4(b6 zub&CfG?&KUhGPW3CX9Smh;zP3+ol6JiN=M}-`v?VM_CTG9-R0xN7>9#7VOleBSNAp z$czE+1f`8Qg0M12SuyP=@G8z6W#fgVIeumgcKMI>L_Y z=76L5%s*Da4dz^WC86!H&ytB8a2=fa$HMvE?AmAJlk1?#&G!#||J&ff4hCB6D>nJh z1tTn}jqZXi&*R{*%lu1<{h7H?WB=S&*wS$Uri`{6h`)lQ5X?c6g-&n4!R#%tn zTczf5VSQ_>UTUsyR|=JWgq1^di>+1*>mC^DR5G{Nw!d4u1^flCB<473bKQY`XHb}$fyqjIs*m&BBMU&_H0J+s?mwr zIwYe$fcZwc7dT6{uLX1d7kTvjV&v$#gr2G80sbvy zy=4lEI!0~=@H9iLUq*c(bbOgb*Y|*%kdbPD;&YRn@Gw8!nZS(j8HUjq?Dl2H&n&t! zi>}O~EB_v@YnlOy2qmT)`;|gw(FI-u3?=MXg_T)!WfolkhZINnu5Z{cr;t}>(Us0Z zDYNKGC$G$+D{{EXEV?p_u3l{wpS}Tpusy@E#J=UqEV>HPdP^b+$S?^PAj2dkfDBs= z#4f|u3}Nerunid|840A#0G5IBCRPcQm;XwjQWyEUQW+Vj)W(7bP0qI=i>{3NfWZn% z+sKeTR7QQ!@NpE-8(-3ZD44mK?EY5_PXWecUl>{%pUeJFcC9Z{Fec)08~E8})CbbZ zm@*EC)8&AgKL=-~U72YYSkB_iv@4@N=si?sXpJ4AVxujrMk7TQCu!o8vb&n z+$=3`6w9m2#ntkbU#a*_e|`HA+IUr}8>=3e?vyecug`{!*Q2IBST9!AGU@}um_Yau zGU@|AqdrKBJCac!5U3WTD_2H+zI2=mlyXgE)CXba*{FqIMtu+{D--(#X4D7C zA+2#neGs`dHm+m~IHNuwbUP84cAKg^qdtI5Q9{DxjQW6GWQnUoDmI)^A0$REFmNkm z)CbgW#K2{cQ6HE|Bs1y*f_Y=`03HFVgg7SLjQW76Kqy`YkTT?ANf}0Cp1uwav#%n{ zh60r=T)Eig6o7kx#=Hj@(7;7?9MlI<+SqcnvXSXdGu>&^=SB~i?ljY#+IV(@8~CJ~ zx=eSP=}ui6Pj99>&2*=X3>R#SfLzr?aXAP}X{I~Pbf<#E*UM;QcUrrCMjM;a#*WrT za7G&ogF&*~X0)+MFC;VCSf$%Yr>cxLw%+N~A0C2dV!d(S>rrlV-3TqCjU|R68Eq_r z^#?SopIzPQA7b?1j-oGnUaP(Bcl>t4_jbFzJ#V-E(C>JS`tGg=2)ka#?;Up99-s&{ zTZFr|z6+G^-LBvD)=I?Bfn&>Hc+mWR z<)A683ht@R9_0eq+--sP=4sjpfkN+7aNW#V@dRxyov)+Qln!&{02Uj+xe_#o<+?@3 zIdMf>PhOvZwjQZD3%H9;sVTZ+uS3fCoF+Iaqw{Lq73sMuT&PDyPtL`3kWpaFJ;@TH zeMxkw1a)u#;GZ*kp1O`;$c}I_&s{W5Q*_5(Z0+viIklX!__Io`L8vHl3Y7uS3DA^a z=nt(SKHyCoq}it_E`&>v$=d`?o@Btd- zePo)X-P8J{jboRar6(eGWK-y*a&4>*C+7~089|G?@L zE=aUaaXH~1zy9m>-fzZunls<0dlFwYyuBDIuw)j>)!X%Zu!UX4l=Iyt49|n6q*j|!t5iQ9wItBa4?bbc- zptIlGcQ3ma(FDrfc>T?30JTK9m*C4tY4qzO>&~w_BXB9_Y`crM7tj@TlL|+(cC+4T zvfb+HMr9SZlpdk#K;}a&DO+Xy?#=yM2LsGYzvZAY@~TTe@$6m0^nlON(8yL&gonH6 z(jvO#)f-K3`>+j7xKDT6z0Urw$4v0t(Px~c*+?rw+iJeYX7}W!XV4tky~7^hP$uqw z*mKXgQ}@3{ddls?cCWSPFKvWD`JX{g)voBBB6P)*gF_JfS6VVp*QPnV^aa~gXr3X1 z_+hcREPgn+b~iubzJn?ec`c%aezif17K0Y#2HSh|$t3zCm?b`-ifE8*6HY?w>z@gu zlS_!%<#@Y(x9e%!!ugzALSJB&3RTVXU^F)mVbryIs^+~gKbQKQBAgP_=2IHr9XRS? z7oBD`5eL9|gx^DmhON8`5^%@JCpPHPtzr^WhIO|O=R%fipq5oO6Z{Fuwf1efe?YdQ zLkfLdigCUf9_QE5^K2Z$r(tS01dY^2=N;|;D|zH%G#?ptC(sy;cO^gB=Xt)*^NHB! zm+~iK%-ELmvGA4@C zUklK!R)D-Lg|Z}LqQDoV@L+EZ;tTfkVpYzVC_F}kF*-knGbW1UN(E4Ga!usf>t}+? z1FnvI4V{E-exu%70c-Z|k;iNYonXni`5QFZw@P@1Db99V_x+$#YU8KWn zT0tkSuA{OqC>#+-u1Lm2aj*{{_@FlSJHb?LzpFU{V?fcJywj<-yCgq+nhtT*;{cGoA##P%@r$Zt7gZ4} zqGClEKveIbNiHhFc+9`5!eSl5tJ!_ir6)D&oQa#k_|H_0~HW!Jj0X;UeS{*9P;32zWk5B)Lj53b`x`}Ef}tX z@b+*F7&9h{jEUmV_rL9Xpxy;l8_HoYU+tjD&G%U_9J%>dJagl)*>1!5;X?%I-3zkg zC-npY8^lvxp17JPSOqHMEn}h}_U@W(nb~;rdAMN@jOUb_?|IGGz&KSUZ86+X%zad& zHg$rO3zh5=HkRs;O$)Vp!r*l)tO31}Qjuxi{epQ}rk}V1854z3sZFez8e~j&+HJ4k zn#T9y6H1h#8|Ca-d`gK>a_56NqT_e{4p_%^!NwO%HzA?C$Sa!D4n(-7-`%=u?~+?1 zQf5E6@^xqAoDp;o%}A@L5Jc#y2qP8fZPGwcc4I9Sv#FOBC1auhfkugM2=vMOAq~17J!7JfSFXBbGA4>dOcegA?>Dwfh2_#}V|}@@wz|H&wbf`Ym)5o`tJT%w z8vOq#nJ9{@YhI;XDQ8R+852bp{oNA!%Q5tXYHaO&(5W9Bz&yIQ-}HCA@PiWMy%NG30c4B^){AgsDsoE40UHiG}$zm3hD#TUC4|iZ3_NYz(u!KztsS3-5%M}-@lKCn8I(0FQEgL{3x zhc2262{ujLo)~mT@dEuFCpve~vU4xQ$Y@k6ChPyXm|_O8fZ1iTfUIcRf0L5J2MK6cP~OZj=1F!%L9SV84*WbkN( z4dM;ZE&ibsU6l+zViEsHWv=YMjf(LC%>kgmN7{BxPt7@<$>tL`0%Y@QO&R8^ai(m|SCj8R2k+g-~D(4|JG#IULJa_%wpy zuCvfnB2ySmJG_ZXC>ZN@H_kLP(o^h0QMI6Ep@a(_6-tcFjf4Lo96rLr4ta1!eR0JG z5}TLhU?^kTY%OD44QDR;gNzc{aEd7E#>5^;*^Okbz5QH!%d6!o{znc{=W2EoG9GXu zqa^LAZYkbKHZ)THD}6Ci3+A3+1;o#UJ5;DQ%YLOyb^N$ztYdUQCETt!_&;@!x;n<4 zDI=0heYlM=MGT4YzjU;Lx>c zg#o-*~5{o)VZ;cp<0_!GJFA4*mzDnv450ca>hOwNGgM#!p9M z8hbJRoEp)lioj19V@0+Nws_&i=m=i+@$}s(JZ~#u&)X-_Rj{fET|kZ#ll;ITccgSX z`N&N=#7nOyo>mi1zooAy+xJEmN3!J!^`z2G%g!jWWi)S~!qKehh5LwWPA?p-b_XB^ zUPlSBhFT;N7phXWf(PD%e%@IXbhX)w`C_HTK@ol&zQIrnQfqi|ny+F8kP-w@qLl zJ2-e0nYFZs;r<0QTZq1@sY*{UBo(=Q*fpAAP3ab5r8MZ@gr>%xn6_dQg&Fsh3t`MP z5v#5tk0I(r<8m3EBJ@8v$fXUSRVJkUoRLiwtnHYNG|+5PXgaS;u<62jPC7hHQM5Be zG<%i~Ab>Me7%+Akjz;rM#A>3d&xU|S_p1&%ODmzK%hgpEqqPfag)c?k8eB{HCYn}K z8=5BSm-zUkRpYuR*8A1U$|_w57b18hM{~lR`~n7^37K4=+ZsZP{f49VgdpW4ZFa?2 zBCVDwl!|)#SN?f`T2B^rO?;;;d-h{I;EcBaz*PItC{=D9Gay<3n?(7 z>Nvhh!OD^n3>K`-TQc~Z)>JZ_jT6yT)zvZdL|GCuwOT*5+G?R(z#lm%ZdV)Wwu!dq z=@e*yq<2c)SO;3O%chiN)l0S@sbQdqh+=dKg()56gAfWq$4$6zN<+$)N$}7{@8fP z!x!!bOGfy`i}F!qrNipO@D%*n)kl4?&rpdSNTpi*%o8^)l`1iNC+n`Nw63nuk=;nl zPDOt++<>m~X82ic6EsA((rJhmEE=M`%NJGjub)SM4r+W=L3Honkll@b(CY1Y+W=R) zMVKqrDytis4hSJgq}B6wz-CW;Z?Nz8{SH_^!uMG;-SHcT9Uuefc|m9%Q5+gDebKDc zG+=$O1k`>5Vx#t(pfRdI{1s~D296x316%BbF7`Rq2PL<3RVX9oub<<9_qoMW&ItQA z3V-keG#O-~f)a?G%K6|aEkdx+$gTZAP@({fK5VeBeT1ZYfvbGSylwG$`Ft4oSh^1s zh1?SAq6CILX2ddpsF#I=ZF#gBjST3ETmTT`Qi{+M`nGfiO}2qjIRp9vuq@FX<4!N? zj6pzBGN7-XN1>ho|2FK^XCAW|&{qcZ)dwpggfNr=eWie7%7DI72rC2nN+GNa=xekP zi;S58eHqR>F`KE7yw(z)+R{N^YCv)s&{qcZ8y90DMgiGH^pe28I(a4wg8e zItZ8~Fu)0rLBpw+j-I-LI~q>CeDu^6pw4jWm7}L#wo9gZ^wbp~$Z*+LkDhwfUiP)Y zQon_!8WaN>bjL#hm;QbZa4mR>3cmDo2J=W|<6BK+Dbvxzd)> z)=gsDFfA2^F$?H5{BIqU7pNrALL9>gPX%ho)Cgc_MZvT(-JvJ84F5~OJSH?_vrwfD z&$jGu`Y@SBF>O8tGSlSrnN~;H10;~TVj#N!Xh#m4gm<+IHps5dPYF!eufff@q9X@l z9r?>j=BkooP_#0`WRTEcnE_z^0-ESBVmPj3U7P~PiU(lOSOda7XtxAc02YSV=kZ;m zLxY8=HhmAtAi}tUcMi=M7`^bz3bqUrEpF&Lme$rVryg%`s}z1355j&LK5F#i z4m8Nw6eP*dDG@C*BCCxvuiY=9)AfUv*V;SS_4nWww(CCUK1)mwyuwy>yHVd-TV8F{ zi_4Ym;@Wb3z3ea7H`W^EdbwC%Z>;r04sy|{d;1=_AN~ONGl7!xnUSSwgb4Q|7~cQl zeusqAUI41JFQJ$%4EUs5ZmLhX^s2b_2GFOyhU`{?tkP)7l8oy{?xAZh!G+k3m?W98 zBZ|30XBXPFTlrDK~Tyy)-4+e5Av^#8sy+7U^4mPjN@x!uX4%qDN1x>Ui zcd#Xc;cdzPs?V1EG;NSzQT!D49GPF4yVs{-8Xl>k|0w=B_y-_6EhG0V`}u0C+k*o)0o}x((_-jFkz%!N|7?Yv zK~thq&ECTUpB{K&L1Tw}9+In_g)&d&Kbdgz`JYU{I{PCBgQr7PDbkyOSOcr#iY)kx zo&7!bOE0XDo4rpbQgsvY<%n_8kHMTPVL{wkQ}M!jgD)pcO|Mqa2G&je;Ia;nUeI(= zY4;apqUm7-s9Mfz{#GJ`A$pxzvl!kS^ofJcgsKtpH`5maz`IbL%tWPwmIS;D({t5| z-V#G9*^LfniB$t+&N5?Y`G_MV1UBCxcxZ`?i%+n9n_Tyvi4b|5L$ftkccLa3IIwQy znK>`?1Odq)Eaf6Q%t12nQ4SFP+;M>A0}j7wmYHIp?TYMW*dW>5K>`Rt+JsL)#)Lwe zkpvEs>KoAC4t86;6-LfE;0B5l;mMZv!NI2aQ9|CsR8cbQ6#X z9S4&%Hm8u25Vi)=66S0Soy}Cec!;;Thc^V)+ax z)i%1U>>J)H!D=FQR&+Dff2~fu?O7n(p1bH$I6q?2%JV!b#j1n<6c*D^2BvN4Q=vF| zrR8LGEgVlRY!K}j_D(ONF`}Ij;a0j_8limIO1Fx|U%45HiDtT8NG$yZ4(1QkWfBaw^jnib^)0@3|54uRV z`!=Zyz8K9dXbXImJ&e{@Mwud_l|Sc0ja=KQx0|~_6%bh0^U2k!f?rcoh&xj7JUy8K z+vDKKri8EVv&gyQo^C2}Gq=y8GqgcH z@*7~N*H_`qF}U|^st?I;^02xDE^Mr_$5^=wXhsxR9Z#e43mE*&3rc5rs{~zFJ72Qj z$2!OUwSyM>m%-)pBss0qLgCX?&|+Ma@kakdEc51)?Y_xGF12JlVS>t9ooguq?+(Di zYDXL3v?k&eQ;q#>9y(C=C7X&I`*==d`*DN7+$MGJw4X96`?guR6d||zM&+q!YJqyVnwE& zxZrBaW#zh6Pq{ik%GpIO*~G$T6ZV=P_S`e>)cvp3@A;kE%yMDr@w>=9Dg_MKUSw=7 z8Cy#ieJ`;yc(qW~0RbPo8{O|Z&ZD{1-8-@~xNX551+$6JqMOTnz|HE2%m>_A`eNk# zy@Z}I`Foqfq8jW?be8Gy;dYIv-U)xIIHU}yHgX8~_5@GWT8pNF?^8r8X9UKA}y`1;#9V8^I z%m+M$TXW_Eoht9YvyZH)bQ<^vvVdF4c(Twd!5_0H*b5Sq{Y zx{7ng{a^!=sp1c~Y+#cMPGXG8o=XeCGOn`{qqPYQ4<$J7VqT;LP=#Qsx9RyWC(WzHDzO*Z8WHz?^iV zS2=V7GX&ekPUupHSyH^CWTM&%OmDt_==pSPkMB4K?az$I_pyM*|M3#~i*e0tcCEC&;a=jV;2yK5Rc*EBbM7L#!ot~S)O;G^ z%%QXF>A`Nj;qUD4HvJ9+pGPy1V6QkoXMO+(@#Y5r74Kd^b1W&{&a(iyyp1;5BW90S zC(f-8N*y2=@>mEM5BKi*Juq;+_0Y!T_pThvQ&ti)(kd-O4WDG6O z&lyqxSbali6+(l?(Ig553wZtMcb-QOEU&n7O4>U+XVhmWHhS6HFo z5Gfo)4U<@jdy~YCf4LJ&UWyOUB$EMng$aMQbE>SQ+bHJ#JEFMq$sidYpedFOcfyoz zr92!8M4jr46WSm-0%sx1ks2fn%`Cyee+IVJO3K1ag{@*NADLUwr!f~ZuYV9OtM(~4 zp}r?th5H{$M?}^PtR9B4CLEv(c#CfbRsyHd_&kbI9*OhV6`dLuQRb=0MwHBxUJMw9 z*&q&pju4sAnmw3!7j?UDy?eECBgEfXKE*M$|i0%mjQWLgsWR zGXJL)T9yGFNF&X!__~^eHb{fKEoOVo@0Az+DBiF4g^-8@xDRZoXE9G$wyjz@l0R$* znO3K_JFpP(tq1j&;v0=tyXk)@HW&jM+fgWxCEk_P&5b2fbxX0y%7G1Is)#p|r@2Ba z5=<~y>2f97WZq&*}T00b%f7A zLOPy`uoxLrf&MY^3LDL!sx*ZntD@Sp$*1^_O>Y zhUle8ba`86h+d9Fm-l#v=#@xxd0lCUUX4VTH<^pihEMVCdA*{+;9UG_-&3yJU!}14 zd7)9!PCTyp7$jqBM|=@m3-k($zwaEw!Qb7a0#WrYN7v1yfb9gYZ{}Nmd|uzRt9j&J zMlt3-PVHKL(!CmkRmm|wujTWx(HNv1a60M2?`d>O{sz4!%Z{@Wz%S5;259%* zf#2B!MWo`B7JLVSQc`yC?rhY1UeO~A2}~U-UoVOTK%pu_OUBT$F|=X~tr|mDjiGDC z&~*_C3UlU%QDUTx#5V`3F~b44w7?*4Q42nUPI4(Q89XwFx$CNDLP51F1T++|1nT0x z{He#_1^2#{2{jmgp9L<|L!{!o>s+X;A~17gY^WI1%;kSC`n#6E3lHSH^~#!e1%#gi9f;%o8q!urg1$%o8s2giGPswJM#j6wkr6 zBY1XcJBZ8^uJ3rvJmH8(y-??xebq8HRGY8b&paFIujfFqDYS=$j1Lv?OXTg^BlCsB ze`$7iQ`VFMGYXO}8o^x}R{b{QO(LF=?Ha_DT7k^vO{efl$=x(Omp9-V3(=YsLg2HH z3E!!WtK=Y~F%1KSkD<}ph~B@V$n6gPP8^MCi10=6<^60CnZ9QvGPTxD&QOWXJu-i` zo%(^_1-FyqrZRmxbk*ou7{3>vP&n32>(Jtp+I5{f%hoELH=Ho12A8S@cjg9X0v8rX zjamURl2M9obf#Q6D^gp5=F80QB$cXQ4|ZG%Wxgf%i^9DB?-0*i!S1n z4sLMU7KH8aupKa#SrHew%B8FugM=twP;Pw13koQoEXi|`yI|MKodkwR`gJ};)$s7; zWJ*k&Leu105?TuXFz>#jy;x=c%Sb{(%qXWBeV zbaqd#Mqj0{JC^9|0$+{378jk}?5oj3I6A~uOv;^DIJmZJ^2JMvErlE25~nbJdTbn5 zo27DjZLLyW-te3C<;t30UtV7=ZZB7w+lAtGX?3Gg?@Lv@iOvb0VsHDf-Jm|Z?K)85 zY`AMGcdZxj8Cz|1s;fm040YBr*zC^+Z1%Bu!+tV}{wOEBVPQFLdaLEN&xSW_;IT`n zk34Ov2Zz`a`T{e$WS>FRJP*87n}_XYz1{N^PhH~cn)V&A;7a`taqvx>5BUBXd}R@s z`ExIyS3v^q82L20I&vfWsF=jA+ioxZN{q2Q_Y@?#ss`1_W&xhUP&mc(z8SV{y^fww zP1gi)JvH_JD`3ou(R>6S=mZ+0@veX+Yy9(kpXU>?&ok>)&cZfS>(&4B6#9cv^i_44 z?D~L90mNC9rr}EWp!Acy}DQm_KshJp@~HfL4b5%JAqE zJj!!}CkNh>Ppj}W4^PM7DK#k-d;56@>>)pPF4J^YfX0f_iM?L$fUiAN$#33(+J5X@ zBBi%I#{Yg4#YS+?TM{PxaMTh~)UdFMgRkqs*J1&zw{mw{Z34Mh8gpi6XP2gcbQ5^W z+;NmE0i)YxbRMWofB_BmI}dztH*OOy6L75n$~NFiOR6$hx)w{jHi5=*&Kb=uEu&Mg zTx_@Qc?X^S-oAU;y@)1I4j4wJ0r(x|UV<+prO~gCtUJHzj4Zu`oNf2|?dQ>#)t*Ir zMDjjWDsAAu8%6O=Wf%3exF!rX_gMCsvhuyUbec5)NT8I11~?x1J4>I7l|M|bc+?q9 zEB>D+;=ghG1@skF{A}o?kbh+z9~yO5PsNTNDckY8H}`KHh$S_#TM^XWQNud^>m4k-cRY-C}BVYC>p6iBfoSHes3ljy1qDriKk z)uk`@>-_O!qx8i-(^TvvRy&r>^}kL`MYC3@WK=XGql5$ZfqL$rywj<-yM$pNSWRpS z13B_4k?y^S7HU^?@Wv2)S;k3I$T(>jO9AHYNU}rOSz2;rLuO*4kzrBW^niyXuc_-d>PSlUd>~)W}(ce})G@9*9;LO{{1!yTQ2D8FE1F%sa!m>$BiwNe?_V zM8K!$KL=}ebeSc-U9cgVmCO=9g}roUiJwAPnI(P-VP%&1eJy{r8J=BU_FqPhz!|ct z`u2u7Ly9VANa!Y2ZXZ)vmMbg@%9UrvNrMpGP*Z15tQdc1miPr{DMkOvIBCqb@&)P9 z5vJ2A4MwMvyU2m7R_AkSXpWcTN_~P+h{H~n&pk!r0u#!~ZB z!BX=r09gD68tBqFuKvt;3PlZcs~eT_XM(OKy@4*NT3Q3$xOZ#@x^eH=40Pk($qaPC zKCj*LKkT_@+^PFtBj$Ox55cZ-&tJ+6bOW3Qv#VYPFWcCMzT$V#@8;3}=AcE@j<4Ig z*RF#luJ3)&sULvHkXK!=RNRwjg3gZwL=`NlCl1^9+xs8175iF-5=^Xb`7;1;mA1Lk z`c^zG!c)ci_M&{R1W$D1+w<}vvCNILy|o(0{?DVx_RMbPDJqRcg|V!i{92S>OY&=Z z=`6GIL?~Z6LCkEA&(QPfSVK=R^}yp`XgfBZ*4FbGV(ab|%>F;{w zjbat=8*MokET+m*%{H(V=X*#s;+(&0$?3CepIJ)ZUu$<@YxFvLE~3-AfX-VCLeW(V zAoZjxIzRf(zj4sNJB8BvyA?}SaBeCTT>uSM9DiT}0W4)DfX*qQAhIb41rb8NP!RCU zxn;C+uTyWfz^kC^NuOY^7jwlH*#tOXw;2x~z)YMhpxc zvidxF#u!&?Q5qXSzujf@tc)!R%+iIqkL4UT!DpFDwdJz2g9duQ;t%U2~%k;`7!Bzp%MDhoxUC?FrcKgdwI8x-?eEx4Iz}xSK z4*us3dNQs+{61KkDO{4V$v4t9m7C!u`P5RmV9P5@o4DitGE#CE?CT^)U2e(tG%|m? zv~nS7{jP(q$Qo)TNFpfZx?GlVMN5XZyg^2`p=Y{i#lL1OCYuCxQ6=5e4I(U{G+KxX zizqH_3cAOOCR4z1sO zk3WKcSvf`|%A5(RJEQ_Q_#fxca(t%TDrHP zspPv~PWLXCw0wV7zE6gKt$PSO*92MbWlAYZF5&}zK>Lh5u6z6uo)8#*ZI-eoeqcK0gl~xkRvx9?6{57 zAGMY4tgA%W$)ZcTbD(;_a;f%WR3yq06-495j!=55vXdp46J_T#tbrt@mGDA?Fhbsu$gux|9^X58yv@x9fsKjiA9qj!686`!!HmLDQYDUyR*C4 z#omWR@bpcQ6iI!Ye5cWJu>e*C7Vs`W9`BcIC$X*A72AoOShPDZ46_N}LZTzDhZ9l1jy?O2v6S-80?O^Ll3X1Gtj{sF)Q@ue)Er ze*K>Au^V>cy!9gnk(5VaG;m3L!8Lx|5XKLP9;ZICdbgcKEH!ECw*zacf|&nP_SP4O z#0<77J4^7)R@Iz!NyvuZG#jWPdV|TWb+eIj-FP2#O`7V5X9XZhb#a;Ps-WH;B%wt~ zOhfurTitqq(uMDZTr5Hg;ztC=-V}vTVVtY0ZX_~l4g|gmW7ix?u0P5IA&GY-->suV z)=)iu3PfZhZD68cRyTo2a*qC{ZZpR*>6@o@nWxflm3S%>-49palJ|PtAmfoyYu)v< zBh5xTJ{s=(201AtDhL2G-j=FfdT>$d(p#80U5o)}ZE$ zX^?oyf`rmFs7WIUtU=8hNnmw9ZRo@>o`>B}8#-8jvY$MvTU24fg+s#Y%gdd{mU&SA zWl73tu#;hhr6C?Ip@m|rp}(-oYF@A*zL+I0DOqZT#)iR`Jdfj`OP8Qx+LW$F#gvpT zOvRKeC#JoUxa5K;DeXB?&Wjk>l0JnpG%p<|#I-7QQ(RHWm>gGB>ZrJ)qNk|7PR+aH z*{zV>@(y{M-Tc|PPwPJ$5jrC^u)+?6{;?$cTFgFbv(~PycVv}cvLgn2`HD2Ss~heZ zPd;ol1~&b^=WXl126nJPTwF4#;=0T9=Hn&+A=o=YF)W3C zLL9|Hh3$dj{~qvgK(!FqclS@SbSe-8f~j5kT&#vIl<}P*dLX)2cSPr7iS9%Dnzj=3 z^gz_F?ufQyiS9!IOGNZQG_dZ7F2oYuhYpsA=z%C<-4R`kB|26Oi<-Ih(!^-E}t;*qcL`(nDC4xQFOOy2pF5DL1X_$z$q^PFLW-aw|CJkys)|m*96ZC zKq9~|CXN!LBX*_X_LAh@=%FZ|=*3%+yJO1V3ap?C#1LEbCN9br*$-9nl~5XW`*u*Q zVb>8nuR2(bJ6jRH1cP7=huAG^#5y4PsrWJ@lQAgPuxl2($r=u^Th@rx0%MI##-Lck zu8$0yHFT!2@BzgkcFP*EreLg*$>^Fj)A(G&uEp#oYdFMiStC{+j5RVDgJKQ4?z5Y$ z;SjrJjW88ptdYqW6l>Turrl%>huAG^gh>HojZ8)ltXahtZ+1N^DQgtRYTYs4-$Yk! z4ys$82%7`O6B&PmCsFzx%8vy}L5f#q`r-!%)h$1SMFQi8jDMH>2)I6QtekvqCm8aD zfJ>YTE5s4~DI3%4guOyG=cvuemRTDq`!ycp}Uh zC{JV(1|lc2jySM^qv458$&Hf}Nz0YC3cj4kBn*TnvU0f7Jke>laXgXKTZ|_%2?ODY ztQGDwPjqT+98V;@7UPLb!a#T;tAjhu6P-RA#}i4R#dso<&?QekXAtS4oN#OBQ&h;Y z9rNw)FP=cup4|GfA^R+a(DNV29{z}VL3UdA8D!;zHUQ)(# zUL|9!j`Z#r`=8RXe=Q3u#Me;flOZK&w>uLA?11e^+iN=57#r*Lf=u4`)oxY^1thgmb^ae+e6XaKjTf>4|XG0F0 zW8^?}lwKu3JHhoETuGW`SCm*+DNXi&NL9R(L z#5^fW#5WcO*7_8&1lt&=_;zOyH(YHdcSGlGfN5+rK0N3 zC7#UaZ?-PWV@qx)q0fb4Y}MSAG^|xwB13R`xYnwL^tC{I02$O3=~9Gp<4GDWat-gk zQ}_}U?Gkp1<1-t%6(veZ&Jdntrn1wafqaM@l<%Zizk}V1X01}Qv**kx`KPea9#{zl zhGOFJuKKub$&3-VMG_IMaafF)&<*Q%v8yqHC~;L}2gu4N?xn6h$qh&xoq2&rB9x*7 zN>unu3!7H_pLB-v#;TF+Dt5#cAs$R%oDbiUR&XgAfXii7I}wl*$5^;0L6u_&N`^%V zsz%f40q(K{SB@+w$*NdUxs3bbFW|+lpsNxbIliFOm&mf(6(qjqEJ=_`QvXSiVKcFkNWz#QtXy|-chKNH(^W{) z=0@rnP+T8(bxE4@X2>m_>f&>#>i(Dg7&%niY&y;L&9%ll4S{VwVLrar*;;qzmzE0k zQoTNFmrKjD#oByhwoq>?fbR%lC!+ss=LvbTebCu z`Ivd$alBV6o7;{H@_ut`)mg5st->pZW*fY|=|nFIEw?w;*)K@`GAUrL*S6NT*POeJb_?=_HtTf=~u}s42=Qu6L9n41-J=_JVMSgcv@aUp<~2kg=9{`^Y1Z8*?lp0 z+wJ|!wD(`LZ$j_$Cs(2OpKENr0KGrgU|I&tsuAzek;w7?PvM_?QRE9L@_1rY%p(k6 z2|H^i#MlZCOzY{ySOys;6zhajU^;u0jCnlwa-td*7`P5H6ke`vy}Z3?DY}o?@!wvxP znSc*P0PYS&rpV2?xX=eZbWN!H?db0Br9)o#W_!vHAcl1bN8Y8cN96B%OrjjHQOUC1 z{$Og=9E0CukF*K4I=$Y8g-B8~ZgEAU>Bb%B!76*4$%d@ohuF$omSyEe15{QiI39eU zIg(oL}K?kM4s| zd-B0RHTjs}40l#*t);bwn06;2TY#GRVwj|=LzWW)EuR&H#+09`An|e$1|T01NW99+ zU27yxP%@LnwqGWy3(m}PY8`KD3vzLSu~$fnFSSTw-hr^dKQ(yj>p8eL85eEi|>^VJalq1koRZs z8-H^rHnm+G?A%e$bzULY+1UepCR5buNriBlalf)NxvHB2!2rW9i8kyS)$?FKg)Fq8 z`N4C@;SHV#Gip>Z=C=bU3L_C#%2Qw^)t?!3X%L+SgJ^OKBVd+}fPE^HigqX)bNImTF+rloUv68SS!x&e&!fcAyR$huwt@w73LY0?aqSF^!rkeyEUvVs0)4^CRS9sCAni-Mi=a^~RcaNCWl8pq^N^ z&1Z-ms$5VjtA{vJGi-%6OXNbRnWEYZ$9xnpU@C%nft(6eD~U5SfHMpl!Vsi_o(`3p zch(xUcB|Q1aiFssZdn7$E|R%W*|Jkxs;xWgDn{(+N+O485@;J0*2cZ0ZCoV_Av_i| zc*HaKWl{~b2Eh(ovO*}uH}^4eDcl@neL=Yt?l-i$t-&T-eStg{hFy z$j-60yh0XCO#bH6wx>oQJM%+h{IWYWy~d}e>zfY^Q+?#*9~wu)4|C{KysV9uEuc?} z=+hGVbRK-E)!|mdnX$R0#yuKo@5+r=!7!uuj5;PCBaLK#FXNpfeLb`IwG3>p z=!$b}ael4#pwV_*$FTd2=E~|;XEr~_Ceyh~mv7&``NHC>x1YH+_uA!~FI--{`i%41 z<*Tngvsgsy(cu$S>w>pxcII*k%jFZ6v!lx8Ha5YbtqEx?=fsi}iD^Nq5m=m}z~rS$ zg3?F|@G6nS;8hCUBUbFbN~!xQ^W9e|cVDH_eU*i%DmLDgky2ZPVFS)D;O;rM-he!{ zbt>iVN)ZJKt>ro=by+Ll6RbS=hJzzi6s%LumFJ(i^3sj#FI>CrP65{!Uwr1uZRfe0 zmv1~zqdiz+&T?Y~Ibb(-I`mpb-}BZ}-n2EpSo9~Yp~b}iS>Kl;qdt13_-e_Dy>-?v z5}0*;$)7g8tZwU9CFP4Y%yb182$`~04LlQh%`i%zGSQ>pQ7_F5lF>t_rSQbEkn6DzzSl9fFx?B=ebashU3t1!~mYprci z!@!`k)!tZxP1PpkTyC`LO_00PMI$>A>VOm(lGjDU`m}WEKt9|-IFeVwbAo-9DVNjt520^;i zu-Ft32KBJxmslm8BJB>VJ(RmC1y^$##s6%ke?v}#D~kxtEahduIVQhCQQM z{gnK(7<)uqJ!JT5tXkb>JmyX;Krp#-YVni?nB0zl=vi~ z&#DOS5T6#J`Lv*-k$_K-p*}p&77#x5ZeNu+CgRhos1A^04mluvJaQ7tCc-*&z859> zpBn_T&mm0f!;EZANJMBiFgwLIS<%MW){Cb7>nPhWRl7@bxy&jN+IF`jOxC_JR`sHO ziBK{9islqhzq{kpKk9I}>4)91;z%d^6qJiT) z!c=gCN9cKv%%fdV!UH^#wQ!6_IyF2>$htVjmtK@{ zoG+L*j_}1lqm}uxE2?*ZFS729@ue5#`%9?2VA{9-P+I27uB+E}jCyVNqF$q10Z3sp zb(kyODZIoLTV#OPL6z{6Wcb>~^-a_L94H*?^TX22Zp)f^;DF@FTdL21bu{z{Zr)5F z4|C{KysV9uEuc?}=+hGVbRK+ahbkh2r>^DD@4$H%#_ZjOcwi`q$bMu@AV2 zWoO7Kr&gCj5qT$b8{1o(+gtiGxtT`-?o6hzPS|1}nSDz2@4ZBx^!^xq! zjFm0pWh+?O3SM>rE4yG{M68H;Gi+JJdjb~sFc<6`+NzDTYNI6!NXY_PvWS!{q9sd6 z$r4&}9w|AGmMkMB%V^08QnG@UTtG@L*pDVT{I6bt8!Jy~Z>*fx+*lbiWo?A~%F_gb zn?5GHZ);+l74!4+CL#NqEtr5haN)xJ-rRClrSEn2ox2blCWMm-`Q3s~HyL)_Lk;Vo z2+9Z7F%A*pyJEsMkic)a;t}{wuW!iZA$SdVZd!G6$d7AtLnteQDW`A)P(v z`N>Qz9)lZ7)VVJ`bI^dqoAh4_{|-CPGHtIw6-B1R<}G!-Eb+0`ZCV1;UU_DG+?-Tl?7v69V)qV?8`$?C< ztTpdK{NpQbj^~Krdw~Ge2DlFMz%oY`5Bt|Rvcq(`uwEtyKy|Cvw&rMnZLe+37_TR$=FvRnRlbmYKy+>5o|uX$X7*DBonsDhx=niaXD&qMlg=nOO{#{+LxjnN=JR zvr0R{td=}um1Jg(a}B}giOepBjQ>i^D$xUR8l1PO|GGO6izvHF(d>#?Ff+kvYkueX zH6O*Vz{JIn>5pF}lwb3M;#Ya+`BjeMS77d9$n?jrd6Zw}LGi1y^Zcqr@hdQWF=YDV zR~hA3r4N2dgS{de=M#&tidsD^yMdWxLEm`3x5ht3h&N5;bNbfp{#%#S5tT88{#%#U zZ?*Me|E)_4p4xh;|JJ4TL2Z4$|JJ41RBgT7Tk8;Vj%}7;b1{%T=Z~dH#%od9c8+t& z^hdj8`m%~Vlg61}PIzsJy249IJQIF7;+gP75zmAdhIl4C5X3Wa{%%n}c&3kw<7eri zbRH*4N~DN~5-sAPM2vVSQFF}WtJ_mP*Qg#tcj)6np@~p9&RgKp!-5z~A~CG^8I201O{jTuW1*01WF7 z3^EoV!8~{iZp|(=mfX}n))7x6{NA)q1@kw}lkQHggvZ z|CQ)do$+5eyKqxbz^-eg4L77AAr2RP)rTU1UDSZC;^x+aiv-NAL$0OuY3nWYKCXr~ zK|oTWgp(yaDVjWkN<+ZE)6o@&ARx7H29d5rjC0XLdLS_YpiKI0%O2jQFX8>P{({uH z>>EAKdmL$7)7yc3U*GJ+Vgt(>c29M=Xpnl$qM0p$RT#<9rA zcI$3y<9;g~ZR`{o=+ro4enc1wXh8X zM%HnNx3IR>fLwF;Xok2}Z3W^7HEHHJh^>%xTF;&*7DU>E1V$UJwFflj-o|#j-hd?T zv~SE2vd>BzsdRRh90Z+yxw!%n_%^mS%+uycGD6Z2{Ae7G;z;^A_%dXtUL2|z-)RhG zpCiVyxp?zw^6{u{rO`>+jjio=i-wL^a+wQZB-S=Mokqtg7K$bCF!1AiOY7ORrceA; zf;(CEuui`%d-z&YZ)qU);bJ$iHDN(r6E2bqv1@|so@d+Li*EHTm<8VKaTb`-%!0)9 z2L$;yf6X9YI!aE<663%gl*WSW)Ii27Fb9MKL6*oPuKX@FJ2VVrePi>%MvFa(3J00z z)gZcureY`r-DPsgZP!Msuo;VHlZ*@v`4XwPO%&pr2su85X0eZuC*8)1@r`j>#6)L4 zNoL&2C8RPWnqaX$-6(f7SY(b|a7)i4r2|p9+!{1wr0Ty14nu- zRdg+dVm0box=EgLaaz!|6bfb;-wC(K({7s(I5I3qy@ry!LW-X7RVC}158o_7=#9_UlkME0PIlJC&)Pl z6_Z~z=%im56Hzy+mlS&PadMWmj7g!}vOk8rqHx_MEL(27e$jC-K)@V%&!#JBXKQLC z5SX%9;DIU4mmBRB2x)j_wNbyjxzU6)xDO48bOdvZ_y_Z*wKAKKjcW3 zQ}h8?hmi`C{FPee1bd#t>bTIId~Qxt2?fL}<#mW!RRI2DIEksmW2vuT2gmx1XB3&=p5kds-R3Tz;(4J**!X3-1 zp#93Kgu9hhL3@=|33n>1g7zt^67Et~1?^E*CETH`3fiBnO1L{&6|^^56}2<1`(%P@ z$<6xB#!BNe&u*`+L88F=s);@gg!>B`@UptQPhhhf*{p4WR?;#-V|=SqsLst@%35y* zz3h^u6f-zomqlv8z#zn5}V+qF+KqlA96rm%o+?p|<|k%8RfVn9}z^KUs$v zo>6d^PpHNyXI&-4%{RE}<~HXi7n;IxeysFtXYv+psj>7d1cj$@dFS95t_^uCU4{g? zkkg8lBnVi3$06%48sxA7$W$L#M+F0Y{#a*}Y79T|{;1%F|1cri);5hzjIG$_k`(sSa1?Sy6UF#jEJon z0~e+TVGWn>>c+>+^ey{wAnp@36(b6ezN4`X15>Z5imP&0+tF;MSJf*`S4YFNPPko- zDU2#Q2~RR*q^tN+Y(XLv0W$PI#C9v_V*xt7=!M%ew^c^*%zJikUuT6*C0L%YtFGq<)bPjFXd?1VoNHBLqbF9I2Eg5D*UKT~9!e{Rt)@U0DcK zv~Z#L2~Q~0eNad!x&rPA0z#OISA;i{q7l3!jaoV64HE*DfLI}j0!%Z1`3K~37#Se| z!kJ8^$QOV}2>19(fowFfzE5D_Rq?=k{C%&-)opC3#}k0``WYcY#U{c2U=;QOQa~&@ zPz~#273G*d9@T9YAQY|yM^^bnjD);!0#r%i9jtHuv%GH+ivZTQ3C-~;8hHQ0!3$gc zRhvVoe-ncB`WPWb#VA1^)BT>r`>?{Nq8-!6MzTJZwLPhi6R3L{nyl~jCU3-sgLQIm zCnze1@jm~G*XOFyDAeb@Vd04YLaK_OeqF~DzYSRy!kV8-POL1%K@SR_tolhpF$mqy zmkz{SrORtoKRM(U;=K#sD=% zX)Uvubo6`DsdfWCi$KUx+$bE9lYa5>BnB}|tElQnG`3F&%qL+TQMGpWAu(Kq4=iS~ zq9{pB0!=ZFO6!LWqCa=TM5gb9Y_%hBnfXhz}^_CL#!;zr%^f~kC_4a`Mk0cB{xO> ztS>97Q7D-+6gq&I50vS5DQ*}I3)B7Q+XaI4ytWWCk_e0`{IBQr=k|a zsR(h9)lV)CDjkCr2VDKM6R4jtL|R~|=CcE0=)=UJ8!tbVat#U|u> zgxd;k)GR~le#8bC(fx@$HB~Ohb-!vK)OCL%;C}ZbgsF%{a44et5sgu$U~t`!D~yE% zR?Z0GAjBmf{D}Dv>-~6VTa~qVpZ`^V$g6h1P@l(x^m`g1LPbYo@(uR#iahQR9ZUE~Q2%tz`@M^p?;?5^GZJ#|XqBCDy{p>sbiEq^((he_2o)U-BO-bi zQS4Nl2h+Q_LRUzj&}oRWw!E>6nA))3JR>!(J^(OYo;<25^wHeKTnt6#Z~$?O@){$9rOU;cV# z@oO1B@B2Kvn|r^xwd&k&G*?!)IIbwJzMNwVU*AUuQ0tuv|W2 zIXkLcZetUIO*A1s?wsRaiHpRvAk_$5i;Du2mnsQLBPqbEL=uBnDRhrmvHL2e?yJmq zU!~lAl}h(j7NV-ycvnVBZ4rh|t>e@-n{(?8aK*1vDR=LIML|Mqxyn?Rwemf|%Fo5Z zI^|q>{+TN;-MIe3wcD2$uRh~kUwrYIE4Q8JZeG6eyamhsaXw+Wdq;=f1@Eji>#&Q& zo%j<^T2Fa%*8F18pRtA(6aQy@Wgpzja3@o^m*M`kEXAF_y3Fq#RCCrTWa9OG`mJdV zm7;|350k@%`3DGcWHA4s6a-9Em*SU+hf=UIaZZ$%AW2UpP|{Ngmh@BtCOt)=$!Up1 zSsR6BqtI*=nvFs$ptKcGXay8n0fkmTp%o=)fg6~rI?`VX-$%**rL70l#A1;t`_CrP!|J7&P1YO*%!Ovet|HM6=idb3=*@+zq(sD35n%#PFP^|fAT7!?&MXl-sDx( z&g>%wj+|z$Sdh}p6~#K8y{yK!$&pJ@Y30K2Xwt0k9F46$c20Y#QghjoCQd`9NfUc$ z7rFBuGN+l^TYmN>?p&wNb|&+=Qy<&&p5+diZ%uBHM_zffhv#CGpRKFTOac-{5Ql`k z5|C8fJxJ@l{p&)O3DC)4}P}Ijh6l6ql zD99ThiaI1Y6!ozW1sRPT3i8H>q7Fw6MSbi;K}I5ng1qses6&xMQ6KwIkWt8?Aa8sq z>M-O`)W<#)WCU_3$QvJuIs`cs^>Ou7=9oD}G#38Br0e`@mVA95IedQ=tXIyRjim=p zxty<)Y#+MI6dZk|V6l<{HH7TisY& za;qLEM?;Sx6xO7PzJMH^_zOtZX`UdHtf@e>X$bC_HeV;V*sCoF__xunwcE7^_iXo6 zl08{#cD7a;5WKUo4&hh)GTRU+3#=gg5iPXb-dJY^py5dYbG^2;zP;wK0}eO#=y`IE z!Q`*i0o_qtNuLYRAY#)eL6JO=sS9W8AY5ImeKJz)v1p z$RuW4@#DW2Ebfz0#siov%zc-amR_^1V=}L*o9&II?fMqb&L(XiUGb(Dz?)=@fmwtZ z3gDU}{ENk(147l$0#tND>jmQLd`4$!K?jx`)UbZTAO{#NZt^!TK`2|rRA|0Tdu--p z`_BbwZ-->*8o63M5|pJ>X1eD`Y97&Hh$+Cif2+B&zR`qU;a_++FlrE7lg59B+apXJ znBxt*i7FT{6h>a~BQcCMPpDxLHPbNQEjFSAOqOBB?KDSHA)G=F&>#r{>;2oc)~Y#7 zKT%5T9@+*EKGrr>2l9*-Houp$wxavU_jayvNXuA1p`ipZnWc|KDvRss8lP9>E$zt0t6V(N0W;wNx zjDl@#K`u@(_Ih$BWdlQs+o9s(1RN<|2o)EHNJ#NwsJP?`AI7~DDlVKxW*%2}@#gU$ zq5iZo^NHA9t{K0jn0eB(s04OACcX_4r2|`huXNy{lbiW*V+S_==1y#CVOuxJ0-d*E zD+DQ7=#Q<&XSU44=Gfh5XeN}KEK^waj(Ujr3c2nd6eiQ%Y08YKBuUkC8DgG}Oi)xk zpBXjJMPv(Nu_2&XCbAd{1_GwSkqA@asm%U}x3iE<>G8zrhN`oWIjM`{r(lKG8!=Bo zsw4f|8<4l?s?GpPY)uLg%jEmuuwkr}nieNaW@?&~4WZ=oPm`}1erUAo_!Th6NVz0STflOlWv+n;Jdgr^ze z=C1-f7fpTSSqcs75Fy?#un`7-gBdmO9;~N<-(V4h#72;i?vS70Ghq_wwCo+8&Ytre zEvFWbLAMW^sZ4t2paG^Z`Y#3lTkB-h2O_6|=De|v`)^$o{Mpje4vefW|6K~iSjiHY zwr-F^A|dGjG9L~>{vje-45U2-5@?rN$HjXJu`3e78BMKV`o{b~i0A~eH|7yC>Xw|X z(S_PCA+M0buK8tvh8?i=?hBZ7rW6Q1&K6IuCgkX&A>&P!T&!lO%){f?gEnG4_uv9)>x7AME_zHQc_`n)d-u!p_1xz^mWxGYFee3Y%jj1d2U5QR^K z1m7k(_m=i_@=sMHGS(Zx+zSzhD1nfy>Filg7>+GY@KwX85_*W@{xjfq`nbO;!0_5^ zwEdB#zXOoyq{dsCEC|pefshxbL+*I<*=U`R%ITJ<=uil0VATju6K;B6tiw{f%w%|l z=I!wTi~iHo{C1F4_Rh1)*0U-vvx=dD5EI9$H_51 z?>xWCQTz(bT@0E2_%)C6t2`)vRd$|Vl_-7%rZ0v}fBY(={HpZ9FKMt>MB{v75vEtG zhh;Y~lPu^P&-d0iB&GNDg`B>1yZ_en`qm5mw=S*UD%*?ww=OMsYU`!`TbI@cwe|V_ zTbE{2we@mut;6vh+bqH60(=?0`^LJ%8+j{>X9@eUG|6}^O54tHE}8ylw@hDFk!R94 z^UH~I3{h7cR)}ZfL_$0h5n#kK>0+x7L&Osi<)o{waGix+(3X-LSmL2{{Z%}aND&Vu zTEs(%81Yb|=9tG|lxw5^~YTdkc4SI-wv&Q^Qm^*nM z6_z|7k~JwFN{3d~5f3rHq(J42CF z$ikr3SlDchwApbb7k%{>c>-q`1Gn}*H%T6ETyvLEY zHN8D^PCoQAMTifv1_z4RQW4Psfg%N|h*+NkMT$}pv0BfZ7RQlhIOk;-=kx38`TVEI zo3Q#R<76L%s z?%1=n+=30?@=JE9vQ%1HDrHZT30j?o(P*^WjU{KPaSv7!6OJ^VyYXt)BnMYE9C`?R z-_3KIx;T^_Cj_EMHDI=WXQOSxlj>#A13nh%EI& zF`!h4FWk!-m-_9CMfZe=D2S|%L?ts@=$4rr-q<2iFv*(Egx33^6>PBnr0etIhY`5zThq+&v zBV?bIHd5*AEI9~D`*L#yhVRDKhI!gNNk&Ne%8ggY;jE3&0X#lrr(PVY7~g3OWuGI) zvblKkY4Y)?E~R0s+KsL4w(EklFB99L0GUDF7g_|;uB}BxWUXh00k==Ti+91oe5j|CJ3^n+$U#q1k2+Id9#J!wrV ztue04WXSPfaym;0NnA^m=)GTf{j{snLC&qF&*1fd41CFkcel@A1>-e|S zeP!SjBN%WFNgh=wB@!@h%mQ_H&(B>M7|W zagYhUTp@mm>}QvE`Q_nJ7x&)ovRR*G9OV~={f=j@g@nQBk*h6oTGAU{vtc#y!e6+` zRQy_G~+mSng+x)I&;Qmop&70mvV{~!E-Dtp(-Iw-}eeT^v z^N?#0f^-U<*J&Uq>m<9fDBpJS`+|o#nL{pDvD9C%a*k2fiW*Z&Cs)0Dvea(qUYX&O ztEJheSY@e&s4WGD>0iY4ZhcrGrG&5U6U4Y}9wuY13c1CfWhuQm>#z~ z&DM(3f$LoLRcF1?UV)9zMFVa+OgdCjH`>jWW(&-FjC3$lP94a?Jt`#&+Td>(W~U=f zoYY!MafI!FYvgq{Yi%&@HXx+cIz4fKm%}vL7sMt#5{&+3R~G0?u!}CqFX?ZmY*#Au z`eZQ0+YS(Sf*kb_mZFLQbEmEEFi1`khSzKm-TGZ=f60FD=p6gI2APS+!Q}I#8w7$P zX|ZTR-nb}O3-OI;=FNypaADUGH1YyHnUd#P*KJ@`@xOpD2p~Z{Y$y|4=}@+P9MZsc zmP6s4r3YegJIn7*sXI$s*;zgj-^JA}HMJEtS| zojqL9>HhX8+=tWM)9H%SNiOjuJPq7whmCiig5w+Pyhz#DOMUEnR3zR{eOmHKkX<&^ zOG#o|(G_>zkeZjBHG&gjQM8^Iak*vLRU_DPsfU*L+%%%YDBM#I>KddTcLKAs{u;SE zrR93TBXpKtu2)6g<2tBtJKR;{u*%ZyXUXbYOH+#{53uW{u*zP zo=yQ1IBtAnl>B_!Q;S{Aw^+;)Xn#-VhhpCYRiuox?tT(i{g| z7M8Z*s2U=nO40Uc$lGq~K=6sK>(DsA-NKf$QQY%Yp_lPIHyvLf zy>lE-IqxQn|DLZ3%@V6ia*Rcf8=p0A4Hs{)zo^u8+Pe0k4TWysp|RpGhaI`rD98{bs9{IKV% z!h@f5|#7OV0YnQezDQWHnpP#wN9Fu%66*94>u1auN4}9TYilKB9X?SX~R8{jxtv zVb8ilw|_c`^oGN2B*@?Pl!`xQCMC5Lxdi#07?Xe(>87FHS(zzR7lD zWqS?Ee%AHP<6*v_iT&m%sH(M&1JMD=LCDURaahPVhh!n8-vu}Pn=~iTXmHdYHyAIB zoCK%J<6cj=9qV+OE3F#D7i$0%VvcTX!ez@QBs3z2G6ziJzwOMNg1~%R%ML@M(}19@ zt*vHl%^c1c5Z~-yLUftT!62byIzkWW2|XMnbUZ-lk;H^fW=!*l(*}-gtUD`>7VvIs zqfIiIqvmkEv9<=(O#pRt;5ePlMt$ZaIVz}Yw_0pToLz@Jm5 zQJgtLj)73t@47+>_d2(n`|a8$lRAmt`DNJ!?Kx!)zO1EXxl z99eIG3r)R4&SoAlPqHzKkD()(N6j%-z^gNWkr^C{k;AK-ae4v9^o%;D!{gZu|RimW7Qw??ALe-eM`$!4L>e=KMMALNRh{7-9wCZ zMNcf13wGk7^M#~EE2V|RMaxC=%Le%=<4fr;!d<>+x7XG#w^we#fQFo+!?pHG$2|9b z`V7gu16RV`D4OJ*j4M~LB9Z;?47(w&$H)jw!)|G~>1@I!efW9zJ`EBH`fY|UTpjbk z!*qtEHirm(afFb&_h~>#w_f`F^bn-jJj5RPHE2PT9H_Ng8!**SZ)-FC#q>}Xo_|*E z8u%f)27ZXHfghr4$S)2ZT)qW7h*7qx8>i{2HuGAk9qX!0*I~tC7BM8PFjZ_T7@Q^f)1~Gid=L}fG zXH6n$P=_;@%mdO5e`03l%tOvvqt+ah648*Ya4aYoFu&reyAS6{HR< z$@5W5a#>oEA-|8eyq?W0#4X9y@RHoO4tX%stI~KOb7TOk?mwZc?nk`Im(I~2^?uMf z`eWV?I!Axp`%$p}Z7M-*=TbRAq4`RJLgj@7g&>qPWH$L#m;ruSnE|SN2C&~x&q=E* zUr~v?${v1ajP4p?tH9!im{H5^Nokrm{4jlxF13SPaz($u)F}D|eu#d7AEIB#FVH3T zDX*VscRua?pxt@d`$4;N(fd)bzn<DjnFY5Wp&iEU8nt`#<@`Pj9v+w$=k{2jU%eJN_; zyDBYwFUt$xwYY`vdU)aM+rDrx)2l(@LdF`v!uPLXjmX^arof#2D@l(UG+O2)Ut`AJ zPlM2TJGjW@W6)YAjAua*`QMblR|J9o3M`qjG95ZgiQZ9FO7=XphGH7_w+8KkR&u6# z3T&VLxzChgA9Tr`V<+fjs(Q>j`hj=JL!*FUmWREtt=qV~3Vvxz|QBaqL?6II}HC~6IN0ERu-P8fL z^e7V0q(_l}A3cf$#F8CF0?D)ypp`ZPMAAlpI@-vmlQi;W!~T+C{il@VV6D zkEZ&uqSI4=5AP7^pFMEM@!@69`>Dmh+?VvwGn^0W(CVK#^nmI^I`8!870_d~c&*7j zM|msi9L15&QQnr%QEGALD0jl=D19HO9nAE4f_5SE`~c2TzG~QCG0dMe$WMaK{^N?y z{siBeJo0{eS{#bdZ`@ZyI(m=Xn;ex++D9Iyhv_MM-%f4hgN%FJac#V$e~Xqt_9_3= zSp3IHkLScR`K^0cMZRGy{>K3v$9*7Zy#{Urb=hrTEVCrL0E}iJ5!DuWoUW14%(A(! zzOmifnwcWw!atn(XmRhex1hb1V0){<_L`!-cX)f?H+L8O<{;%f^YDvkf42GrKi5k` zp5u27`_$#7+WH(j0Sq4lYKq@;ONNV=t0mq#J2kW~;H`&_4mA}+a6%OytDXml?99=& zaA@wqBT_Kbx?Av(>N#l9KV7wt@ z>MY5yn_x~ZT-ug|^(67lkY-H2dteE3#$ShHxr=jSOsxb^vDP7C)vHo1CH`eAyAl)ebcQBG_` LlFgu9?nwS01~!j+5ALv#eU zOut8S@{dzzSw^G(yUe}*mEz}P3#g9SW~%e{JDEBujobWgr}E{7%(X_8_Xqbv!TFQ8 z@x?vVU3DL;r0%s?%{hgdq7!&fr6!d!oy?D~y9pyMH=^N*Ca^y4#hZ7Gpz5<1;rNm%e8=xp z7&|r;g5N9f>*zuaHfy20_FF-^kvq(Z_u*}-GCW?-ivJ0{fng`x1rxrH;NL3C!DF#5 zic7R$>xQo+xc?J6#~;H}mR0b3+fGm!WUF9ySxcKo_r67CYtzy}gm+=w4QKgARr z`PzlMtHoo|fHn<2(1}0e{dl#>SpIOF0$u)aIJ*8@gKL8qL1M;!;(MY9<_K2c#X}EZ zVM!I#dCjLm)w)#Asu%MAQ>6FElW|PKy6kno0+YJ-O$@F_=|n&PS-)@~{6U@$SLt^w;SE+@HH2lJD1on#eFr z5ROE@+8fOAv;+0eyC+PN^1!QWK9a#aFVqJN?-ed-tt>-x{&%k#eQ_32YI z+Abh^;TOTna2~sI-G>`>nbE3TUG(b^=jHFM>7`NK&Tkj$bC0n}WPRT$h`%*}Cd*vt zP7^D>U$p_fpUdz&f9&b^5C+?3DDr;HfDI>&aO_4mzG>YVxcF}}ZxAa0ozH%B@p5Gv zKh**bg(`8(Q{;is7a=zG7`QFIPhyuYq2uQ4#k=P_@rhSFko1di;D#lCorTcNh2-Y$ zFNjOExX3qWtX%}qcR`){t-TB`2S-uW(P}(S`wV{6m7@FhyhM{rzfpvIhVi3g(D;cK zZogy7fA(0>67h{d&R63I`#kX7;sLEnY3#V(B(y?bT9>z!TC`A@5mJgildE8PavnA} zjpnr_xZ*m1xJLLFmnO-s}CXl|(%0#gV zqo}z4Sn|TpfmVLZUcF$mg;yRvjn}fXgsKmop!*0v zp53Ma5B@o${b~#P@#q~vxRQsFkWK+8;8dTJG1NIcyUB{J91s6Fm8zD;!!Xy{OB=Zz#w>NZXi+7w#lN&xAH- zt6%5I*+a4<#kv$K0;-67mJY3UHRbOt?dWmYF5Gyl73at|v4vVHV6f@JkbX3h_KD9> z5aEM8JEn2F;$0AVeJtO)!j8M>`SH=uvrtam8a;M6qoMQ&dfd4e3+AZPYZ8{+b-4@; z^^xX^C++B+?bhJiG!=5{j^cBtL9qIxNBe3N>GczO)HZtr?YR_0?=G)^H-@QD_o_s& z+h#H`UU3`GFG>YP#Vo+e1Ei|ioKwwgXU(7v(&yQQ6CMZhyptX@<&7*)oqR;ldCZ6h zeHq84B~-c9$lLhG<`SeWY!()ppCaE~-U~i$Q{+dd#o>Cl36xBkNW)$`Vn*`_Hgbg* zfBb2guvT1-^8ig&+j0U^FPflEUjiGqtORrFLvd;SW1KwNfq>3xemC<2lscQk$y=wv zOm9CrWM*UO>li3dIRU?!fLq&!35!xDqj7I5NT%%tx%3)Pl+%Yzy1z+h?N1g`dlU0N zD8Qm;?hsIt4C1k^VDd_dY_`n6P!TKKCEbfk6P`0Q_p>-9xDcmJwWEGrMfjjX0JjS% z*7VmAuc&N6>Job{6{1C_NDg4$qGF5)N2rW;!Og+R_*GensohORwA|2I77jPu$X)hc8TT zLyOT*p|vRGZal%;*f1{8k4syYijpY>7cLZ!gH0dR(wfCKw}4>p=|LFrt2` zpy+ES9Gx|sbUBR1PZ3W1R^f3$XFH0q^!>y=m%MHQfy(b~K`3(C1j>Pr7 zcHC|4RZ``mLlVBa;;zy{9Pw}}pL0$W)+_1Jtg#ZHk(@}PUwp&c^Ueyt8WuV4DJ{Wm zN=x9|m^H%H(Vpmkz8^l!p25G4OMz`!XUIJ77BDkw0p;@+cz+j#6T=E&rQ2LMHeM1w zukK<-A4~9*fjC;lW*`PLcyiE-{_!{>)KnXQib_%FB`evpI2mYMH4J7QOv4aK1DYl! z&$k8r6GRn!MTz61u%|bJ2(P=buXzixY~@8Pxw;;tx0nEYScKJ^C*W~sh z_!yd+u9F@|V~7}di_H(^p&_spPo_kIkMu{jQ$CswEUADJIZ-xi8(`V)By@_E$EG|^ zeiX%%u({`1q-i!RJ8A-UR}$gdl4fSNuN&{r$;YE&4LIwhKKuyKZ9DL1siaLr6ws2&BA##^DgmAmDqZ?W?jQzLlNBM(p8^ti>;$+YW*HYP4D!(A^Wxr5tE z6#qIO-r8BYR z&^$Cxfwvv=V2VgFIlb>0Cf{$vV(mEGnkNNU{~W=xfmTv}t%`KV2QmHG51GG&3QbyS zPd&fJK%;d9{`-1-=vySRNhe2R`N+FyKhupL(-*+^hW5(HLPx&pNeKp*Gz&jYJ40q& ziRA_LZP2?K*V+nSy?u zFG+111*@PxV7_Jqc8ePEk4jTv=Gw)OR%M9Pp!sL`Qo%YFb29CM#7GtO4Ks27RHLcX4#fAVE>S3J(A*u z)AbDK(P0mXXy#&?-Mog;qffx2sug4`r$d4#fv*c?ux~*pEU{RGmNW08lHp9YGg)Di-9=u0k2Z^%$WR4Qd~>`R4|TI~Nw9%Zc%rA@P=ci->Wq`Yc6j$Cu-=!Q;Z^ zJAT04o9D>l3#UQj(p&gkd6L{~Jd2J;UxB~J8)0*<8-%;|fXWId{O{ikSS6pyb`1HM zL-q|g_|Sp6S!{!n3K?p$egW~gX91OWx&#$zXK>U7D^xu73jKz+!|t>-EYL3<#_0fm zea{k2bio zo!$mpN7iFD&4GBEQ;>cCfS`P)g7B3<09ttm@fI_ppVqD9KUMyKO@TJ;&lzBj{#oR& z^ikm^sUj8;VgxZKL{Y*Z1an%h$J_H_%XAN=5a`*I)1CwIW! zlhMp{wlSEek79!s;h;&>X}W&2VEc@TT%a)uxbH`S@zfV2`JOx5Gf*wO8Y~6(Rq9bK zxQ4AHu9e^HCi2vjYos?b4Hd5G@a{1M2d3yu%J|x$J@r>T@PuDIC~r7a(>~x<#t%Qyn^jYKPh2 z{p7>LM7UqQ62&Ah6Qjvmyh~$mWy;(SuyMjEtm?f6%dW4aJGxWZmZ|RaMq&XWPaM&; zUIu)2SAnHZ66D3p^7p&-X{VbDI$xIHTe5C~Pw-_duX-x1DL28r&o{%;8Yh}FEQ1Ur z8$-Qe9A@dI;G5`hGWLf&owF?oBb;L^V-mJtp7ADh=`_GbiGi#6?{nBDmqB!HiHB!f zZNcuuG5Fy$6JwhjS**7bJbTwJup;SfQ%4&_Ec}Ga1FeOPdyYVw=0udVQb5)82XOS^ zF_)AjK5*<9R`7LQQ| z43Cdjq?)r2UR4#>0f!YqIKHAuxb}!X9GgF%57viJn+twuzbzS8xZD$jWUav34fE*e+cyW`F=j)@N+X>X!_i@geV;DEAnwW-k-kK6-=7nib@w;u*XVSI91(9Y85t zSC}{2gfBX60>wvvlUScB2-N%oB3re1@1|_nwf7&f*mW0XSSP{>?-1H4Iu{FE40xd0 zQ(~4L4*kX($+g-)FfAbzXBA`t8}^Z8IVM9Ssb-dCo?vq-5A%Jl;FdAb=(j5r+scRY zgKjo(+(HpPP2D9twc<1JvM+!O8)INx%`Nb?R79QI|QY+jkK4t(=V5^ znT2jc z&}y6miNzVz7l|lI<4_;+7ncX+= zr;AQlqWbSLJiA$pZTojfcqQaJ#%Jc@ZMB1Vu(+5Nzp4iFO%w3xW=-lkV3!X!Z2TWA6!Q)AeE_C zULXp3BOx$Yf{xK14nj#Iy6Z;)xDKv_!1Q`t_O6fp+k6ks?!1l3)zeVZt`XiO>4Eg7 zi|}{4BDXs9kmN_lLx-)UaN_Q09NB2em+P1d#HQJSZ(soF3wnlOQ*|M^K!HCi&SPyB zPb=H>|H1L?zuAM?*{tW#GTbm(0$i_Vu>ET8IO+0Q@<_9oO?_m_zua)c@aHY~IJ5~> zFc)wBTY=i0E}&^4!)s(Rpkwa`P@i!EY=Tpz;Esitah~mblnd8~Rl6AaZrBZMj})_8lfhC`b?8^0 zWPHAIwm@PiYclNcp;?I=!BO1}-CuTM=cFyn>!N_XTXz)~n?z9cAD`j=VoI|>eVZNp*^!OSCf+1Mi(vf> zQ+Ak_vG&vl!a0_oNX|4fCTnbkt2dhBsio1(sQnFGdNz{{8(q(i`fG8SkzG)$SC6xY zKVd7bq~ShUDF}rTaAVypm|V1zrUdCh>ZZN8^6@J+%BGr3PpBugHSGeElw9y#pM+aJ z^?_B$WZ=C&Q6ZpN*rL}AMaOHg+AvedLNqb(!7v;-t`P4R=<^L%5*YU6lG8TFG33b! z+%4x%(2`?O$RRw`+J?@@WKntfZ}Q|+Xr;`>I(MQtb;fMCFstQVZ}=X0%-+%Tr!IwUbu$C ztDeA>BMLaC;j_~o;S<5;8`&@=@dDfb%#b;OBwDA1z{pjX@!&EwOzmCB1O4WZ0rN27 z-+?K(`BD`7&p&}g9$myfYmI`9*J{ySYYBwYS&$dE9#7RM!nmOztaqm&-5427EMjM4 zfZcK&@7e`kd(7xA@e6DmHGyrd-gsVVbX+7HHLV(<_WHQ z|4}&qPlG@+bsuz%+=r)q$I&*UF8F%a4Ccs7@{k>`D?{tIU=}$fd|6kF4o0T*S;STL z>4OgX|2c%e;`d|K*7I;lFW&BL{)5=?3rUw@mv*zwm|B2e?p5slA&XaqoPKPSUB6_-hRJT)PN# z_szw3D>SIaxMWiNK8^|omXO7#5>ef1H>1YdSW~%ncrZl z-_>~DEFYefUB%1;gYkgRN@14g3*tMc0C##D;t{h%kbl}h&W@YLd_J53muG<;+{$YoP+)yyxPc zXdPNvGl|uH+Qd?Ri}2;U&kGZh(goord&gqOzLgkKAelk#q3a%9Xouxm)c@uo5O{Bk~+i@8Et z89`agCG^qLHgafQG=9+6Mf;}*n1_xTjX9G{>ZY1;CcB;_#dSllkq-CK*vr}{44u`a zesl=$g;)KLaA3kn@YT*nz4LeAoU$%euK0)IDU*19W&}8|aYV^4I|R|aHz2vjo!pdq zfI08q2|V_mV((__vjOd9L0ID|@Z7W$_72VSt0@V%B3uL09E$NrKp*BWav&0R8gQ+7L;AcY0n^1Lpyo+G z{0>TmLsM@OR@f<&39Q2Lt3-I!!P5}*Vya-@_Cj(Y@fduq{SV)T=*T=s>O(>I z?qnDU{DlVk>tX5(6PCYg6gNI|4K{^56ZBp6#g5?N)V3m=&AHQpYi#rIblwd9OJog; z{AW%VM@d2B?ikp3T@F_Vd!mbOI?4?a&?=NC15dM@qwoG?)ds_O-j`iu^tN>-JsxJ~CKNf453`5-?wqVPMwAdE(~apk&nkG`Rt}R~#zDM{P_U}h ziQj061$ucUR9<*Nyss_6u7tCoFJ;f325Es|SUc0To-w3fqP(~FDooFAVG8C_e1yL| z?$j$A^0mo0GjKoZcGtlVx4jUtsEMUKxCrOQxWKF{1GuM3jNWlxheUTjj!|~uZqbuL zWvv1j2Za#Zs8)!$D?#75L}Nwa74#Xmlzp7XaJk8QwoE39w0vrU!dd0GW!haZaGlD} z$xvRC+{ThbXX2NsQZz(;KEB+wA2Py+@v^U7=ysuq*-IR74)>ged6qjs*X0r`UNZu8 z|GHysbs#o{6=BK0>zIEc0P|P+gP4Z6@Wz4ReE8#EIHGkLPT3?uO`j(TpGg6AXzf9> z!x=a&UY`uQp2d~tq|kMPISnarMAP4+xtCu*d@C`>a}keNTjybZQ+*Mnj5Gxe|0ptf zy##$K7sVp(x1vRBB;0wB4^_J*>1d1j=(?@~R5jDc59djE$(dJ9cI+VC^K(crX~)f_ z$8ng|3gWilJyb7n0Ufx=EQa>g%zDK7+5kKxwF!lrexTR=By6i=OtM3ckNRx@om0-? zealZ^I4+ctmlCvJ!vvnUHVKm#q_ES*UeM!}2aUqv>=lk<6}^LG1#5vmktHx z*&{H8Y=aNqW%;=FGce3dtvDL0nCpZuLHeQ0H8P{;V)>!s3 z&ICQ=Re9+Q3E{Mv`{2P8MR>1Ti0=&wVbzZD^zs`EJnC?QOwxXY-oCcZPu?w~>&4=+ zO0rKlGSdMpZ1$n;&7+lXTa>8QZCem2u4S_lRj5VpVeq^;hJ9F>3)va-xY!R7{`dKH z>{j22kK+zQ-!5mlyaB9}Fe zLxsg5!Pp-i@Z?YvahfRuJO1r~TDlkQ1wHUo#sCuZhWd^N+3;5{PhfvWnsn9#;jeuq zIBmWro#&qG>?GY^x_SvE4fcLuH_`Q@iTi=`y3^L;@KFZBDlI^AKqV`36tl1 zgt1s5G*w}sCV+>7Tp)Zn-RWiG$g9%Rx>q2p6Dbh_sZbzxapo=k<4j$9^FJalPsQXL*# z`5brdG66Ts^vd$3uh`suSD9z84FugQz~R#_vmbtEVUWpD^{Qc5LcfFT?bptizn%fI z`UE?);RbBG>_iI-Y-sq1a*U~$q3x{^xN`Sj+>=)#G`l*MN(?6BLpeM6PdR}!F$)|s z{61zD1=E`52za(O0wQG*zXX3H6LOSkK!mkWB0!OM=c#eE(WdaNbSKVx^qHk4Ri`mj}(lqf{dsou*&u{lQkIwAWA7LZgN6FIJMBGES)MDoG}8(ZnhLcH)>* z*D+{q3;e72icCKv>gt;Cju5S6TJgb%6dHj!sNaT>{ z1#>|tW{iz(TXDs0ReB`17_*-JzdKOIyS6t_#byL7$@v8fURc0jY@N_MVHkaKUzv~8 zHK21eQ(@wDvYks?+$9y44{4XbKEBpMicSfJ{^ffwEeA zt~O^FuD_#B!-C4;XW=|J+3^!zy4?e9mqL&hjw1FIKgq}pH}2hUNG{19LiM6;SZdQP z7;g2M-TQHl>1vB|6{}fTUDgE;H0+#pV+C_0K`XYWg`>_>aS!*zf6+*7_ zJ~HoHy-@1sWTZ3H_=O>sBHm>Q8oi2vItLE%ZfRtAm^ClOR9N)g9;@zKLgzU-{%ZCn z5?6i(ekfkYXZkI8=MljZovtMF+##q*7{Hm12LxtXQ|T&P&W=tQ!R|f}!D~LD5ZK^@ z(Ho@bqwY7t_Fcu;ievDZ{4P|@a_8GOcncRi*o!YSJ6V}rC=CmW1|ed|*MQ*ZnS4taB-v@Z_gZU;KIKAp|~c^lH+%0?`_s0}VTdUX7_U6{RB6qouX zlWe;Ul|HLQsJq`vFbTVX^FK(_y2eJ5SW-z=rqz%GlTzTuQ)L^SLe@rFGn5;Q0 zf=x>r;8SuG`ES)G+#FUY@Uv`$NJB+VOm|lknY?fx9krJ z!$>+>v)wTHCgS(O2~2#`S!}o)#fEjyfD`T$KyIxp&rFe{84h!()TNC-fNlXX)}(tW=hRnK51@FkKS7w~3+p%*{|aGXj6_+>IAI z?fH`V8vLoBg`m`Y1d;O{Al=6v3qsLIQ6s5`teUROrH1<4^?gTJ zg{v=wu6m5m+U+1a_O>u{+j5*XY8YHyJ%*Ajb)x;lmS*d#;kN0vY}&$noN;&+Xt$ro z{n}Be7IY43k4nSt5#h)ltN|naOK?9nl!(8Uf-U2V(bYH+f~J`8oeLgg_?t&8LZg=j zxBVc8td2t3?yETJrAs1D>|AXO&wo*6C`DDT7kU3dn{HMA zjJiYF#5#$;cpx$k1AkwKd$RRV{l1@s8}Gtx6U(9e(_9oeIGRV<)WaON*=&3KEGShN zMYoLF#5}xDV%RV#{#!YiTg+>N2UlM}SzDgaqP?G-I6ImdEWOGkM7NL$yK?cCdly8{ zdBSX;y#Pf#3kMqJKxOq^?8;lj`fdlI_O8Ep?ByomB}D}qyYw|KGmFNR&#G{hs0Y2J z;fyug?jbuo#EPuliSG{TQybf2qPul6XuO;~l<8a>$^yJ-W5oiRpsJ0M3rw(Xc?`>( z-U{}EvzJP7JA}>>=3;7+Y+<2-=Xq>10-Pf63;P?wA*9qw zAiwS>3bQk~=$9zGspd|njc$fhU)$iQxf5S?cmrfU3?+YCdfD|?56Hpi#_*%d7<#X% zLG+Fz%%jH$)1y?m&yN@=`+5c);>`GyW7$IAM~fiBPl7)a{Q+pboD82iiV74`AYoM` zxcXdz2IW*xN;M{Y$2Ppc%SrHd5A=vEXLo~63eUc)g$bYH*wrm@STNK}zg80i=f?_k z^7bB(tec0%V^+ebxe_$__9ZA?y#>occH@6<>x7$w@8FhzL#Tzj*yx{|p|mp(WM}>) z7sx+!3{wGVK^*=|v!T%y#e!=NZ?NptB24|So`t=-0ypF)&;w<6Ah{s>Hcf*u<9EK zQ*+!+>_e^4Y&IH%4-+){09RarwzrbX71=Kx?C$X2-2wlF|LH}h*Uc0jZ zI_HOwv-x4LZRQPdU6qOkN+R%aeiQDzB?0i@B)ZJd5JKEp|oG zeRk<2c=~L3JNSaB?OF&nhV8gu`g52+jJlig_^>K{}} zKx2gD5NGJfM=z4Y@TE<{eS5U=gUUTYe_Hc79mG>%8BCJD1+BNOXpTl5 z>AhVE#Qy`i`o{$tG>YNY0!dmHAVuYOf5XylQ&N|?TWz9|!yYpYr zw|@d(uj+_itJ*NO(+Gw~t!EYwx=8;^H$MK~2qEkP%mp?VXYkX!aPx z%9r4>{a5kbAA6K^QRCO{>>wJ!t?a$#b}YMK1|iFwh)aW_~O1)>v)fOqCSo?DH@d*~pOP&sQ zu1L{M6Kq(sV>O!?;6tYW*bf~lJ=kr>*!cMunf1H|tezPG#hR}L;puPD^xAJ`b^8>! zo{1$+yC1`ezJq9d%#qr(1;B$jpV-KcPXxCf`tsQ#FVX4dOm60L8ryoO@R*Q7Hf)^* zw~O(|Ah#M;0R*`)1Ye3&V_2;tCR%Gy?eOzNx=Rria&O?L={r!{Ef6j?7C@xcNu1<= zjWqu&Kut+WYL~x?b{ucR25TicO7Si zjkx8|Ei_AgOww*f^B~FVf&*f=$iFdL;j!;a{F52L&-6aP03}VhUt)`ID}+qv?OWFF zSq;_acHlToXJ&KiHq-bNLf$^jB_Z>-z^o(Duyw+6u(J?YLqyoEq%KMb@Vyh*zb$>a#x7z*cG(fdO7r0 z*bD7DBgv9DMIL)bAE$p;rn$SL$*PPh^6ckADt)&X)-93c=YIC${FVXYbJ`lFDn#Jf zp?56#Cp5?`=-w>=CDXDE!lk0F6k2rsS!H~s;dy%S)=o5#%b(gEz%@MR}9%cH?Q zYi{m&8yn~r43!^`cc*;>i@afct?(PHaea=5CU=lQsK!^h7jf0DLqj}sD><-f4Ww;q zggFTgaQxv^D7^3sN1_z%kqrl#(HhL-Sw3zy2*tquUl=uZ5oC^fjn?<;u)Z@6)P2*~ z@K_;yp4toJR%r8@ce8QeODry$k%m3t*O-&j0NyWpMMi3##y|Pqba&Eo!M|27GMSA8 zuhTE!+wVj?Dl(g@f6HeIVF4KAc?kX;*oSYs2U+2;2>Nc{Q&6@%$_Dq@VVTZ(;fTTW z=;>a~7T;{$`)&b@p%esl*YR z#*W74KShMuc3%8No5WB*YzlHs$5_g>qmUvO2yaD;F)}I^{<wiQlRs#Bp zx#*pF9HYO-33YN#VpHp3vNk~p&emmv&XG#;bEgHYI@t_Z{17h}ok8;Wz z1<7OeAwB&~kEgw1!3VN25$=%*?F!Os2V*taShd}rPi_7D3*qHXFR zDRoHO3zFc8!`;f%F$*ELX9bE%EAv@bj>CctnK;`&5NZc&NLg72QT%I$wwaUBJvE=m zPgw^u=eFVB2^P4v$sFg|r$YA@Lz<^ljmjzqNXT+u(0zWIy%)Z~9q&fd*(M$MhaE)m zqW8kRm%7ktQzFrrox?9bv1FM)?&7f_=6BY(Ntjz%4rh-WvyFM%f&Ujfl(D80<5p9$ zAlwUl@*Rcp&ySEttKzXT%MnWa2)tkT9gW&Y!Hg;Hw7q&Ou9VC$Ce;g#Uv%K9ra?xm7K26q3Apjm3D-w2pxahm#SJ?ZAeCrvZ{HunK*2^V8q~rd zF*O+eGz#>6o3ZMy4(P~sNonEy;F1V>KPDtuK+M zj?wV?L@DT1$xu8xR(71Hva%43G!lmmZKVsco~Bu7OTRE5x22Zrxv!X%Rx7t z?HJ}2O;pb|V@GEi7+k#~ocr)O{L~-Cim50K9TKYO1BY48sSEhZ&5~A^r-5GL7f{+W zhseF01qWi~=<4vJxX9)g#{8WIi}zNMy39ndIwirzIgMd&W41v^Nid8H8^JWb=d;^o z?y&#AvqIuCg{JdJ2zAwi84309;Lq{Od#w}k+MFF+MAw#n?v%mlM33)IYQwQ>&0$q< zFzG1O#(8TVGwtq1l+#&*`zj|2MjSp0Q!STa(es;V<8up*iXzZG(i^Y6*o4)pL9k+v z1qSI>vrRsd+&v&0W|2~6y73k~$(;tj&pI%5^RxO)MByXHscD1*!GW=aokHd#}*i{lkJ=>~& zW>vS8N!mpV&&uCp)+!&Fzwk9)elCW$L#l}N$}4bq_aFwC?88~bDrDn{cMv}L7KTb{ z&~iyx2nx~Wu6jEK z8XNKD^Bi*LP7~3;8$um6G5q&OgnhlT9zI{%4Qclpp+T=o_&}@~sMH@;;8O#)X8mKo z@{T}|+$$)K^nf@o1Nxu!R5s7J5m|mDIM1#_F)-lMro2X#_1V~zsRE0JX13aD4O~7Z z0TNE$X90yTNbL!K-Wgbfi@R5niWR0@m}`pGOM_8%p`G)^BPzlTk<-yL!4$)e$k6+h z7PQ!JXs%Vgfu#cn`0!0)_%ukCS7-clQlA;ijIQ;tr*RbTS}h~f8h4=bb{Dw3CKnUF zx4?wzVVI$4%h&8qgg*09s9s4i;fMxKoY=*3oo7Sw$%SzG>`y^~-D&*VC`MQMufnPb zNjRmEM80=K;;mn4zqu$p#V>3@~9#Eeu~#f@hXCV&k+c&|SI@HqD;n zbVyzw`qIYGP`)4K&-s#^^P531&jA(h#V~d6tyr;KlTk5b_8YG` zcwUqST}wOI)z|>DMfZY-jx}yxe22(eiea)+KHD?819y$lg~QqxU{vR6mi zm)3-X+r-J3wj!A+nQQ``X%+Y~Gy&JkO$1xF0NVdmz;cqe!sTmHbfvQ#9quxRCQlCK z(mm$9zoHAKNH~z=6<#Rnc>wM%zAqRNrApH*GT_3ICdiquLI>U-f{b~L$c#0>{s-4U zqF5RhEOH{DL*91Ng9UJGpaN8Mu0z@5=@1reNgIF7h3ZP~=mH`~Vf5!z+*0arO{;xMP_LwkPixx1Hu#WM^SHDQjk z_Yh0p8rZ(57?!XHc(P+E4|rw98>7|_`LZu8-R2Nmn@3oqULM)I-WMKc=fpm!nO&ns>jh1Wh+-1s0ZY_YZR(D{{P;TAg>;_52gw9pIfCE`u z1-NA_J$ytT&B8~rGh0f8yUSwXwcH=_(6&l2Zi*(4EGmYk3ni>isS)dMM$@tN7jT&F zF`{pKlsFId-Rrif)1zs_>8*+w$Q^S6L#ZsDygkG*FLK3iej}O9sBbtvWB|na5>aUC zLOd=uuq6#=;Aw3>j1)PDlbUX^aw0~RmJ~yt*IitDp&CplJ|n-rOyobyj-cxIpLl6$ zCCgMBL1yHBAn7AZ@%zh1U~9CN>U+!`A{?Xzg&*2s>9b>SLV7#QS+p3YnmOaxtJmSJ zaU7P+{|1*VQikSF2AlBeH90xq0Hl@t!Ci%gU^TTL-p@OV-o}-v+#AfUuem^ylXLNd zXaQbZxEb#}F$Cq{cW~oEGK`It!LA1xn5(^vdfs(Lr(<96!@oSD?HUb9FUOK-g*|Bh z^&D=myM-mK4?)Sri0Yoy!e?pXj2@WEQrlPZo~@tI?@=snt?-0bt{LE_AB&O>l61ay zCOS1Qqt7~y!NR>i$n)XbaIbEIaMQ(jY>kVDl8v`n<-+^8u=z5Ujh{gb#2=GCMe@}A z)ji>ND;H>Kk7eI7J`jsh({cBp5z%<~mRWce!(r)EoDwTbw)PYsb zwxUvaK>8`JTor)>)?PFzAy#+=;%R8^QHfZH%d1glWPT!qJj9*d+g5&=B_tm9=(~ zKQ~XoxnfOPYTpo1skZZvg0}7eJ_J6FG94gZ=ntOyj^TnBH}o@weAVuw4L- z+WZ9Dcb$N-2h>SfMFi}tvElVSg=E%^TolbyBRhj^fZ9!?V$$Mtdh`R(aT4Y8gEH}M z^=PO{?1HCqx-hnX52{;w;z(ODI2nSRaY>**^E&tC9KO# zi?$34Ld*2!Xl<~6D7S5ZI@Q0#w5}gNRfG-Q8^7bM)?-4d=la81H&0Btl#Xgf=V4Zb zGk9g)!nIyMNt;i(P^-olOVsqZ;ssMact#0#lnvc?5>JM(m$9tRVLUzI>p-_@NTa^V zMs$iQBZK#SY5YAqzM?r5S6YtdaI%yv3<)Df!xnd~8~8o^?=^!J2Qr0y&NldM*m<~J^_PkG6p*9N2ZiDZ>U_J|5Rdvd z4E>9QV6bo|i@CFt&k!Gr*GG%u)ODe-{Q6PQFOvt6dS#SdDoKa!L&&(FgZILq~|9i8<5ObF;3X;fP7QNt2>&h`xOLZw*o)bGAmjc(XI#ZyUSa z!3hfrhu;bE2WlkIV+oR*t0_rJ{)qVe#Tv;k+xhKNw9+T-+A~Svw_?G-s&@X!3}?al z#KK9N+9dIHFP5~Wt+bWX4if};5`s*%5V0!!BmSghA=nYhm)KNr+TSYb3#2CTi$N#8 z!IS3!IHZ zB^TH~1=Fks#Lq?N1>4jpfq}esyYJUd@$7CO5&2GX;b-KEJu0UOYP$RR0qxU8_p;c6 z12dE))TpE6&dg51mL08<*+aRKioSYrjl)jKvKz0(^rwr0hkIjrgNCgV{{B``L|TKW zac_&nGa+$DB4@e;514dM*(4?$N@ zzG%pGiEY^+TT+y6D>3sg<(r28o%D4J1PzK)Y%gy~;72YzAg+3_c5)vz@r!<(kjyzh zAifxrA&|+q$uHNfpVZ-4(zPc+ylr8qV3ENI!SQb8c8mR z#rN3f1haD?-D-t>QJ zwx1_!bvDh@5+pV<+J8izgilCKT3+Bb`DOy(x1+x~8l6Dw#{ z^6gt@wSUU3l049gmpFJ-3dB^L;OV<#l0KTtUlXj;ej#XWd*x)lY@n5G`)ZdA|Alg) zXz*K|_>J5F@%;~zSofm>K2Kdm(%d|mzmKVta7&fiNp6M2tJ+2MdZb&>8sOP(H}i1Z?$~!zNTk3POI} z;>Sy`6^QdDu^2~<_FpO&1R?%elGrqL$)ab9?VpCO3uK7bj2#US-k)5=!^IG>4|Qt z8jH&}@M*bOru-<%NUS1bBsP6#D0VPY6Yo(q^^au0lq;`2eJ^fTKPQKX; z%HmwsgS> zGj5TOuj}%I^b;<4v%(A_m=$ok0fFV{*$IvDm}&7tf+w zk>7MhSzJZui|oel@Dv>_L{Dgc{>WpdND0b{&99a6RDTWf6xYq*Q$D*zvkoVV9)>6J zcdT|2mpgnBok>0*N;{cB$E0i)ou0CmpZwC9&ms=;>)P$bH)6^}r8g#cpH?XFwdN<2 zyT=U0URRCyOBV;B@L%Ed+e2PJEzy%)1^3W#CLam4PdjP#;$-+z-ken1s!Rh7JY;G#{CzRrTC8ODdspDi`_}Z3zVh)Z=`vEn z^OLM1%i$JL{F4k`wT>S+VqncT|Kd%r*1Ca**hZrFj19ae|K!9b9rogVGi`XuwkPS+ zYCgOJ!aC92?j@q&hI^u5jk)5;A|rmmhNHCqlOfTAylmbFaXMZR$`u7{99C+4QoBd^2gyun;LeeHU3my0i1<6KFW z_ZrZt)jee8VJ-=7+rW`y@^Ft{HTG>bByTWO$dX0_+9 z;hB%>OuIQO_53gv@Q2cSmF45te zaiTrfn&9=(yV$A3od4yz3xAfWs>s*Qo>%m7ov0)`g*U%YgTJN7kf;7+gm*AMmMk{A z3Y70;@|JFVPWGCclb?fEinibU!83fo;m(k3p|zqf!N_&#*rLsc{CQ&$yjk^x{Kl(7 zCQB;FyMe~!In6sXBkUhZRL*AVub4q859v>(Xg_uhTJi8Ob zdlVSrfZvz+Z@&fbb57QZ6{G(0V$45_#gUHuRT}pEq1s{b!^S;gJr7#^mHmZBRWSG! zhm<9eqBMS5vl9Q({{4K_Z{|D&xk*mUAGRbkSWVI;vX{8Ln<9AsRENL4sYiU(q*u)A z-zgG-GX8z5$Ksobc`&qXj!5;h5bw*0#rIY3kjUE@l(|jkz58Q;R*wkDgyMMkeZ_py z!7cyD$FhaIhQWK}zu#+M_ij(ZzQ~OpKHo%to4ua5(MK1?w-sTH8`J2Q8?EWbmD#vB zr3cQg*hbGeID=Hm?!fi$T<92fwD5Y77B5mri;}M&V_1$)21l2X&9#x+OwRl~hT&U% z>vEwb;~ikzY#T{o&q)7brK)h6wYTCCYwnek3^U$TmZW<&(?n&JO~sBytg^PhET?D3 zZH5{d)(p94Rv$HLncYQw%%s)&O!s0bmfMxxj888ut&ia-=61u!j6+R9R?4f+F&<}Y zvzC8|wSKHv%iL^X%1oZ$X7kBmiq&uLF&ka)+sxHBv#g&7cQE+}YMI*_nasyYwk)Z# zm(~{oPcd76L8jFyO{Tr79V^*?BWo-#%jQu&wrMH4z!(Z+FzSFxpy?Je_wom>8pyLNCBciPr^PTgtD z&VDz-M9*^Ag+~&&)QKJ3hqn;-d8-Ha??x@o!M*Jq8E4jW*XX<$l*$#p<15>Uuq~(YWNu{GtAk(Y9XN=Cwv0t8@X?>6+xW5yQGedVKG1?v)vGY3;*$YGOaHNtpa-X{J+17idSx+y= zv%ihAIWGM>*qmRZXXb@ua43ffc9QpyO?}l8_9>To#?rHDHYUM~ zSuphhW53ZUW+|`A#)>ipDA?Z<92uO*NBg!^>lCSz*dD z7*=Oq9*$)_Si6K-IAeuP`oD+P!&U~29f^9(zWd)85>tD&WW0^}IRdb|swNiH6`f>R z3@>J?&D(CZu}8wlcRp*i^LH~tv-&AVtf$PBb$Gxom}y5St}SJ1p3>*qr%z!oH(kKt zGtaR)bbfP{2TeE&KQyvy&&6|FaV6_0p}as<4A&#~IU8 z^|^m(>&=9B8kl#@n^+3cPK?!9iCxq_&Q>t0Vs&j)W#4}t%yK$*ih1d> zAM+h=D(6a27CS_t-DXX!A5&LFo%KVunzg@fI(v$2KKuH<6APaOE3=e+OqiZr4(CJW zK`U3~OqSh-sjP(-fsEUB)|@?S7qL&Td}TBBpB4Mdm=iO-lgS~EKDE-3mg6op&|yQ> zLmc+?Ib5)vVtaD0vIpw|7^%to7|Hg=+<6Yc+@`5g?3rJevio<&u%9=)WFMA%;{-eE zvR%DSay~@T%zlP~jSPIpKEO_81>7@Waubr-p27ysvBo7<^x~7u8*mq+cR-pU^(Bqj zk>zdkZq_l@rz=L*W9kMhZi5&2NFInO1NcGHbky7#ZgzNB_8zpaJgE9lZb8mr5IDLX|rQ0_c79AI&9*6_p;c(6Rg~>{9ye_ zH?msTL9i0MhHW0Lw_sn*Xl1!8$a5->9%a!ll5OU(R$J|5oMwho9M*?FlX`lyhs`a0 zIYwGwKVxy=X6CI>4My_jc*gR;i%jX_35G&Y5##5LeaxDJi>)ixOR>i|t2mc>9f*UH z9q{%?HVS&a9ew?LhH6@#PCeas1G#!Wr#!#dVDY+(RIrl)>MREBr9X+A-DkOPv_^@0?%ANd!-k58h(MlhtpJp* z1%}I{Vaw&S-2DOymF`SHomZEE3T-ny?YlgF>M^}7+}8(9`4a#%wa%lk=}vfrT!V|t z?xH2BO!&DfmTPv&9A{-7LyUnj;j#P0#FEt8sP*cV>ebWd>gQjW(TU`xmli= z(D1@{ov+~NYp2MO3IX{wO`4R-aH3yp&c!i7pW)0y-H?&p2XbHj!4`9i$WDe0RqGN9 zls|eCO4aeGw%C~1*LIbo%(4W1Z-*!|BPKFlT}WJH8G$i_wQWVBJn*c)4w=PWCe~KO zaW(ZcQPA~9!rtA1Njy|V--dp06^$Qq4ZE%)t;|L=1GIB1zYS4~KEx2ZH;$l_uga0o zxk_l;6AIQj6ru{R0BW`LVY^iCqey^;u0Th zbm0Es6y#EwN$9QpLiv!(K={QTV!E3lp8MrE$e3MBoL4ZVguCJp%Qzjlir0d%MM4zu z_%rBI^hVP~Gthy69QbAbL+;d_v#?d!6%cuQi|}-A0bUoHj*{ps^iJU2Q5vescgc8*a1{Rs1o^$s(cV_MfhZZ>l#kEmzAWjbXJ>G|_)}DZ? z+CG8}qub!w^;?C9J*E(md8z2htD`t%o-^!x*2s~Xww$t(ZA4w2BJS918`!x(4QF3y zM$AA|BQNIwf+UGNGy)2D>j&M2a; zTN9wj`!^LlDvMuj^@j6TW>e?atOWFzSR!=AQCz%rzVO2F(_l7TN0|&<;l7v(VU*`A zsI%n^*jPJEc@=B~Vc`>COZRbMR*4(-e{BhGna_Yz--g0?^#f>%jXs`#MhUOH@Cf`4 z3dbq|%W$YvD&@RxIl?j&z~)+bXQvC6m?5mk+JsXM-+%!Ink`-*PGA@B*@(GFnHu@d zKxu~~(WewgNQnNz<-|9X&HjMqo)gpCY(uc)#}lN_@};Q#`Y$T+51Y1LWl1kIs3SYt zrAR_5hg|S#CC*8W#bs1HJa90B9{hX|RKGrs--SnE<&7WlIpe!zl*dl8Z=Dt?rL~im z`WT0vrcNQBR(yiRw~|3gI-o6-I!U*l8tz?1Yp8Gfh9>*mz%G2R-ksXGe;i~f&%o=%g~Vdahu@c!qF;xU zvFVK*pl~;n5*=2dp2)`{xabZ_Gn)`j`EnWDj*7;kU$y}EURBJ$X#=7j86ywhyWrc; zG?<}!86CRd&xHjtDC)I5Z1I|f-p{^=7VhweHPI)~bTGM|J>C+oDw)`MwV!Y#?KMy? z3PGu>E8C{@?11-DOd!8l1g?sB$mGo?Q0=J*yA9_Ezh&lOjy?yz*v^C8LN|CIMu_4z z11#SpgSXC*hWqV;@cW-`cs#nEs=qFU8nW(yKcouQi2M&-(Yb=n{M~SY{x_I^NRK-G z%r00BVI_>5Lq1(1Ou&=9ta0G8r z$wUDNy7*XVMN{CU-ePnvVj5m?JOT8EWP{P;G%DIy08FA(F~za~EiV#@_yTt%oS06q zRz@JPu$*d|z72q@KL|Y7fl6iP;-7gf;7`m4LgAYeaH)1gI}UN*`!-VU_oG3{-$bry>n5;t`y-)4YbTg&6Gw2@7omhT#o!Y^1x%}1 z1_r&`ICTl@DE+M4$o&3x6mvr!>@uUNg(1&~FjP)4|B2AX_0iP#nI_2d`x3Ne&vIhK z@)NaZSutvv{X{rtQ3}|ywjT{H&g05`s0NqwTexkHVhAvB(GY4pSYycl8%up;$B7P>!q?#8WC9aqp z1<5-AI=3p36%tmpwQ+Z~CxIEyH=*2{P^jy8gv+%zC&qCdwd|eNBtsGrvq#SnDSau#J4p>? zyDtwp8+~sTBxMrG{Q<E_a;7v8v^By&xFs&6X2|# zNWjrfG<~K$ygT@g%O4*AAB}0sq0x)jD(OWE7x`H9E*JcAv7#7J--s_Rl}O==gc{zz z27J$)hM%(66Q(;ZQ``Mz(W!$I;Bk3Z+pc+^sV&FvQx1z=g@^Wt(Yya9_v0uB{dJh& znlAiGnYNUoNC$atQpYXe6Nb@&r%6b`dKzxH(M&`$vcbEs44{-0hU^|{z|0qNaA@sP zuC(rB;%2W8(%Vu8EKB-;a)K+|eR&j+yEbB%S$b{jv$Wxt@J2%RZnxTQ;~7pp?8J#Og5t#HiSJW3o(SHt0v zYZ0xZjuQLE;o0$6oUp_O&$MeqM7am0@l`-{e>{l{(x!qpcL$+=T@BihRzX#sa|h?H zeF4sbaPI4VA2i;ZhdVw`71ql3pqRwX)ag5`Kw66uN-HYooIa}{j`Q?q)CP-NKy%c;&A)L6H2rD{{`|0_^80g< zS;u~$_0|uotl5h@^DRNW*B7cw{sj2$E`_N#MmVj}4sX3_fKi?ooV%=#TWK>3FL)g_ z+5AP7+OeaQs8ySdQ~p|_huRb5jDJpa_3nPGbkv3nU@4QDnNM)z^c1-D-Vp7v-3)GM zVUWw>m(q5jE9ju#mvG&yTX17=GW^f=5u@=?L<_1y=NTZ!Uo_6Jnhu&ngJAs9n8s|V&2m|h6PXX@tVv*UQF;w_fEL0z8A__dOQ`_tMxr&T2 zFz`?hEnno0u3o!H44jIk_UM+PH$Rq9;pe$%C99RXE|US?op;1WDNm^hzX%kyBZ*+b zM(X)!8+esE54Mr#(c$AyIHlfA)a(|TC=3LVqW#A7xn}|?sChlFAp(=JRCft{;ygSYa?)Z}mPKPpRi(3sS z(LtSTC{)8onmnMw`xE%9!#Fn8`VJon7eI9}mo$46PklJ|E!-U8=pEZT;~gRghGOqyq{FOL!9K-7G>U)+`g=CtA_pi)*NE zVnF!1cQ&_EC`RRa43Cis3Q~k^ATyl_WZz zbcGb#?;u_L6X?JN&h!S8PI9N%fF854;vIh#MDsiE&}%G?kt-{9k-caZuXkbzU1hnF z4m#q(%X3>S8YNooYN2`Ddi}Jty8w3w6=fxBYlKubbR=50MJ1 zl3`wG1HIwdC-P230aY(hp#3byh*cjxfhS}Qa&6hfJ@h^fY}}p*W~jD7)t7Oonp=a5 zmp_9Wp1Q*G7xEx+K7`P&uV~di*@66PYVeiU`7q$RDdm$PjRuoTaRfHDTS%k3jn;4-N;Kqq}S!a6{n* zm_H>4$UpRh6_yUb#dj0lcmEe0ndeOW>WhGSyN{!u>gC|VwN7G znc!Wj7fzE6M*1gjfd}#wj`Xbs))ASg;KmI!@Ax_Bwu>*^o!o)E&w3zWY%d&J5QE=b zw#Un~UZI)!7m0iK)6uRmCbqq3L42#uh8fHN91%id&iouaZ+#+YzPSqi1`A=y6*c^( zq5}SGNQR}-^2GA5d%&%m6X^YOJE$D8114Yo0Vjgvp=*pX?RC(L8xh%CLKWh0VU zld}c5J-Y;sPwzp~r5A|SWgH;)bQ_SYIi-;7?!?{IG)i#D!JpQ@Cpqa)!QuaUah{<* z7Vmlk_1@kBW}9hjYVC!0?7dIsC?2GZjx40_={I1+?nCE3?82MfZ^PC1C*bmhcS+WH zNS8nH!!0+yL5pN-vaE9r2>Yi*ACP^CWGZXhOxhB0{q1@xXUY`F-E|JhUWnr!sJVkw zRgVFepj9xQ(T=`vlf{;HD$vd(75&omg~9H1)L>XSmHTQl$c|cv(s#cG_GLPtdfygw z{j3Jc-fRFJpU1(pKmQ@)&}j7UY8v{ttQ`!_DTctqlW;6o#4CR4!}pQhl;3w#s=&+= zs5pKDbJW&jWuGeUct94SuC4*+&)DM!^(0bGe+#zPUL~@13GDoJ9_k1`Mkzmv19#P} z;6zIa8d@Z3{`7S>xHGJe)?R6&%mPAT4ZH-#o5rXs2drTkvlMOJ{g?P~whJ<5Cli*c z2e9FXY`9~Y8u!yX8c#h@h34y>K-P61z@=anWN>Oep?h{Ku`RP7I+?ve9S6=~XF>!L zyn>dIjb$yoFNFALptZxQHyIRN%z29!}fPD3v@eP5RR$pqE;57G_l}C0&1paKBtXDLi^@ z3s(JWOd9X0N3Dbro@r}FJF0hLCrXu^)8~NHn*8ts%}wO0vQgZV53qBIB{u9X#yJru zaIL=zz4W;OCWci>gFSgj`qxxS7?cCMUM&N3LcWyI2RP7xT_upf=j<=!_WtDRLDUS+~vF;p58l~5{o$S&4Wd#FKw7`UC5vm z>VvR)-Yej4-9$9{lu#>T8j;ccI1qbS5iX)1qKPx|0F*32Dh4w_vh-1yeL0`2xco6v zYVja0RSCJz(kft6U=mVItU(GL@6h^>7pT7%6|uRJ25SFz4bOYA83h;?z=G$u@YHXU zr@SQOVAm@HL_T*rA<3$ONTnOLXlde3y+Dwg*^Z`7n86uGt)ZrMEvkKy3sZ_71EpO` zgtuA@SbvH@VK0^wm3ejOSG*UL@75s7`7M+QLkV8I;t5vdxq_TR0$e-9hHG!-aWiJK zz-qA~d@<&PN{=C^h#G-&Nh}d}sS)+~<$|X^71(DMg@3=u2OJQM^g8bWrew0Efm;q< zd|(^i@F4*+4OhebwH!iq=RUmsd;}$s*-3dE+5vXn+>OI~uY!Q^+wg7de*DBK3DvG+ z(e776xW^wYA^RlrCjZk*EH$V`7ys_W&oVT~2=5Ir?RhlmKCuL!`^ml~D>fV$+73Z)K9F z+++w-w{j{eZVj+m{+NpC4g~6^4xHFk&0x{BN~(849ep+_13Gi2phbIyfOZ-NMMkNp z<8=wuv3(Y}dHN%AF`kYM2FHl8K^{6YF&6-4Ec!i(iFw3~A=#CusrO%c3HgI^;C%X5 z6x?(P&C*y;6W_X+>~3_c1r_Q!T`+HzD3&6xK79gF%lL!R!^Tu(u%*YzT}%vreX?hK{NDu=ptm zqWY;ltMWJvk%~w-s)c_XUIl+@PC@2_`FP8NP$K4f8gL5y!maN5F1!;m177soiZ|zK zV9#n5k@2!sM3oj>Caw?Hz%UZOHqgc30&YuG}S`|z2BMx@(nD&e%EPu{?-=Z+gSA7I$GAq&NuTI3- z4nJVOtc|j)y+oZ_6Uo}zSzdP^I&7J+=n)XUGv*taaSQ`x99PU9IR5ugz zZd!w#Iyux0kqKIoDFsKGC7|WtC}pUA9f+$8&|8IMVExt==2|NdT&2J0f}avh>3&FP zSC_&W@1LWCkuLbuyb^H5Q5SvsiqMv0CsAAdT+H6P07X5pfj1Yu#hOcG!8xn#`10B# z!1>o|)L3T+1Kgj&lF`dKfAr0;uhSK^L7@5=<=l$>mMZG4-FjDF|Zp2`~3luTU;Dcx*osTDMM@8 zCSZSi4f3bQXW`S|G@f^2A6oXti28D_8lAhd5FIN|gVye9ykOmM`svV3ta3MyzQ5ZU zUVC>LR_)1!-&K0(QNN?Oa>^>W&QgmkOZY9^5fca&?KXk(EF-A=xtYG2v4{+dAAwba z$*@a98sY=Wu+DfT^gdgR;O;WoihBZXe(w$fp9W!((HSfnZXW< zAx&J8QA0K+Q(#H)JY4*F0s18(KxStokk>cBPtBvSzmpVfIyXcpyiEnh+D%A(&RN_O zv6!+vD+6c7l%tl8X3ASG1Ft;R09G#5g62t=(4D9n;)Ah0F#UcU6l_t0FH1{W#cOBd zS10S>%$$De$CAa6vXVwull8#md!Nzk2W{Bq;xuUggJ#WJAPeRWYvI+Z%R#2;4ESh$ z5>~0!1UB!}PQ-Y5{Q2q_moAdSpS!*iMdvMruUzu6XY+jAQ~3ZXy@~?0Tlb&< zLJlwMz+85M5C(SG;ccOjc- zdcB+u?bO!LE8L7oi(PD?ihU05Hakq6^_GG?7fz!g<|gj7Ma$8@z2V@(&QN&ZMG<=6 zFzIWs1(13WN`=}6!*umZ>aFM!5!>qLMVs^Q1XeR;IX63b4r2!w!JuKrUjCVr>MeS7c>$29OOD(5f*Fa5`Dq}q|AsF>X+0| zPv_l2ykax7_Tec?x^)&(mHvl_nMvFoUhlXYorjU%{K3|rE28o4YJF5wtO2g3l>vuk zVW>qf92VUOgO@JKqfFlrkW*HGY8B<-xqrK0!zMp?A=(apTg}6EikF1Wh3nwd5L2)+ z$eLK;D2MV43?bj)5wcJC5AH6N#?|s&$fLf7EKJ-6PhKmdAE+{*)AKScXmut#8UxAF z>I}M@Z`)_^*}{NU_f(^6jc4KC%?@PoqKCBq7C(+{nK7aKPeP%ehe4yr8l8@Oz|oq2 z98EhqKon{_6Uo0!k(JVAVs+*wRQ3G_GGWGp1bP;Abp*CU(bTUKN`V*YwZaOqhPRE^Ch)Ay#+wKhv0pJEir!jE}Hhq9u}CZazhtR z=0Etl$ft#-j?m|kF4L5{vAzX;T-QqN{G5f6^$Y4h$Mx`$Oe6+U1IX*y5*&2qeoL(E zX~J~x26!Jv!K!OwWc=A2EEt%KHEl0~XKlZM?Uy9fcO1YB@fW}!cW65o-$g{Zw{mf( zED^Tl9NHvj4aKqroDeN#49G=q8q4IUnD|TJZ2EW0t3RIFUgSY4HY|ytPDYLZ9&HeFcA<#P6iF6+s5TB>l zBL0+Aiqxyab*g5d;8_MFhWzoRTu-bBqVWf_WOQl{35-_`g1xz$;nqFM*h4Rn=qF1l zBSthZmVF3RpRL0GG_C=ekQ7*AtWG^wi$w;k2=wIKe_Ts@89Z%IILz8-3jFAF5Q_psYzeqmEVq+&>|S!a!OL(DLJju2`qEr#6+iP(5V zhz@QMuo@IH{o@+K5|^=n1UNS#bEj9bFLxB3l{H_M{(V&kse)wR?e=Vg2>-!uwg4z zdvybTFhw7Q-L`~%j>oXW)oS$HoWMVPGq?vjwsQ|u+Q2z?cEE~>Ce*K*Lfom}hb&Es zfl>PvptE=%N=T7~5>Gd<fsW(s{^Xy#`UY!Vw8{dO`PhBaZQjnW)G<27nkQ z;py2!RLkEYB=ZtL*svP4Nt^|I`!hhkcQ0jGZ3&66O<+og8d{_6jp{vwM3g|9Ug*f@wrbfV}vjtt&g_FTHFYc}uV;7$6_UmcOGmIbfqc@LQ* zw}JNNg}uGFuEu=mA0x4={AT)zy!O2Y_W4g2N!teTZoe_4-fH_HN1bXS?Ug@rf9}L>iC5y*-;e{YRmZ4c z9RZNru1y3tsGykdm%t9Wk0A8|2fg{?PJEIvr7r!hMw`#{P%C$D26u`MqR}m?)ZC7-}gWP zOCCntw*ZU8(|`kM0vAb#15aZSG|mYJSO2TRQoTx~jNuE=$vFjYZVW~`hxZYCS}x%6 z%Tn~em|d{(zCSuNr5#*&?T-Wh&ckl8ogmTl8crF~N5d3EkN&%l+{%_huC@s^FO5gM z-Rgy<_Wxl-^~>X5n$~#vHH3n<4N=wG!qKg&)wt~TW$y0ly09f=G5)8r7qLCgCFnX=VuU5X`J*MU6(sRQ;*jfXaa_wmZG zcDC!w4A>jt1h{MbiJI#woSbcwF~{B!V6{F%RJXE?R53h6j_8%pyZX1%k<fO^4uQJ7dh3Y(fewh=`poTJmyEz z-l5vOjUC^~-dq>nSgNvU?dS-;_q?36bMGSQ-#@|SfI(Wrq?IhpKFwW6n&WYyB3k`8 z4#c5Qkpd50Sri6RjGUl?vJ-L0s|jhI zP=lqyg+zVYaVQ&5hTf>ZM%_O9&=;ct>Iu<|R9>t?V;}>wU#>&478{{LJi;HRdLUX9 zhBfk-;PX!=Wv`t8rE;2J(gG#C)Jzj>uQ`jJNK7E{rVkYEt4D_xAA^g?W^TRD2!XtE zLC+B-T>R}HS2fU&y38E}1AW&yc5;(goRK-)b2kyZtG+;NxAO;)G7eDpTu9rCT?vRc zYzgBmcM?T&DCjuq0&GVEP!D?$Wer{jUDq42{kt}7ndJ>iPvoFr?oXg;wG>a;myLBc zoW*+o(y{IKQ*eB?7x&y)Htx`$gQT;&F zod=isl)~$U%O~TxBKoCPF5Y^?5_&94hTW$%aL4?UQ0h|*(Ehg*c!g}C_4)dwmh(xd z_Ma8Z`(8w;+NYqfdQIH5GJXW_G$G47OK?*9UgF`lU>Kt*Pd-n( zDhxAgCtU5!LE_(P)E;CG8oW;nZTr80)eh5;#C-u8Z3qR@m95mna%T`3@(}5i@OsCKVCPT~h4#jPCD+cP;%tPPIQgJx znGSAH$w0lES5l!9lQu@x9z2~YOKlkDfq|9*r157C_phcrC?(U-u!jxxJNhGKX7`Zd zO4pzn?pj2VWG^CY^U=@&IZ!xmU=kgWBZdKJ3}){nDb_FZkne{~#u zt=@>f`&|W3`V(PS=l>YG5=SW7I4t*7A&H78M|Ni3_nldGXO;*_hoqEFDoMH&T@q5{ zELY{uog+C)c4wCmNl8MKq>@fP-KFU8`4?v1>$!f9CUa&2=bl^_%C7wLURXB=XC4vJ z^rA76|NiJi-((kH))Z8-3)Gb@w7Zu}-Yb3-^Yo-Gwj0k7s6{JCaGAgEWOAqOvskF)+I|mpo8Mn>T3eM%vsgdpV#pB+UC=#b8m?p4y~q} z*GY>i!f|0qWitC^-C6$jx?QYA;Tnj zC(`Qu#cWaS7JB9Ll~wVAU;N-b`pgln$#lcvV0z@tUD|5!rqG3dh1T#Z=l6b><9F*` zWQQGbCUx0!(RI`!J~Ho(eEwy6hSFNTu-PavU)MF=ts=~9M3&biu7UC&3AK1HRR?%)wXW8{@ zXNt8oz4<>5u*}vBcd>@aDY0+)HnGB)NsJ#ajPc6iGfMt5#kKMmn0k&6^ZnQ&bJc}3Y z6)h4!|8Y$yQ}bzp=Q&H?ak66%RO5_GsR>gfb5fL8KaGC$cOAR<%maE$=S*hNZc~P@ zYQh*a&JjO2lOmix@r`;6*6=?%zviWxbPCNTJeIa%U9oPWt-wtjAy!x2EbiD6)~ftEiB49BXG3-bxBP(BC5pl?s)^Rdti^}`VeOAq754_0XyRG-)lk+&Nhx-A(`jK2wY3wmpvvCXE z5#+|(5UnKqX`ez@F4w0wZ=S=yjB{w?=LI60#xDK_4=sk>7*7i|;`siTA)W63TlAqd zla{&snhyS}&F9rzVb@trXI4$>6J2u17E|Y6(lymQW-wSoe17$Fe#2Tny56Ojt_YW6 z`imNQi8hAffWS^h(@MZb%ywi7FWR&ASGLgar%o1c|IsJ*ufv$1PWOc88Z~L=iS$JdKP54Pi8R%tkDdzVMU84P z7es07y%YuUk+G8M;RacY&#y8CX2rRZq>zbs{gT^)(@FLcaoqunxy2hK`(pn{ss@V$ zSzQH!w45mxLDX)^mfd?K|K$f;6wL{#whPaaytRp!G;bG3j9z9-c9@%4ynm}{VP6tr z(KeAEKR)tNY)`%uY$$v$Ftan_GnQJ|M=D4yuie8UP$*&`pUKs{D42cB3X$py-vYO1U}v6 z?>nQ&MyRUNB$nkSChTccbxpgQCu7uf=9=FX&aZE18R0mxOEB zd2~~)2J`G_9&Pj~j|shPN?(6>P7F7z2~-c6(m&O&GL0f0`$R`&BD!@(Y`;lf68OfG z85v$Cx&+GUtOpy#dq^dg=K8c{Ev-a|B-LgTFxFEUm+foUcndLIL>@KaY}q>p`~~%u}K`ByMoRd z-Or3K-^jd;(i8I(o-mCO70goN1g|XVF&*=Cs@TZfpNX5?${bHu6DX~?%B=O&kQ~%@ z6uP5tj9mFG`po0g{EaqO`I4LFjQ{9$al>J0!SwRQV#Awu;zkQQ3vVA?0rObDT6#ki zbHrF)f<3$;7#Qpl1oxX%*Zql=-0cSwJwVa|vt&$=Po9%_L);>F#WtpBgd@!+7WFB8uY?dM+NpW#e+!K*K{6hGpNei&O0djy!#pt-=9LyzCVXF6`jfPM^YbAALgy)SB=om277x1s@k)ot8^~Jy1*Mshs6~i|-Q2 z9^Oxne7(SaTmDP%XbmN~r7kN`rg{Vd*-j?>U%lk5>Rra6C00VJt`(HqlnS)|%ayo= zLka)%xFkcmkogdBqOJP}Us**TS6ncTzkx)ClMM^W^x^&$N2a1xWM>Bhg?3*fARK zcva>k{o>FP_S&$nNd31WoAvF2sGgg}ZgZ0G0&ySK zzSoWYiLIn7f9i^FEj`CBK8p!QRlo6(^?cwXKNS;+bZM35 zI<{)nTXwTa5P#i~1EQDmwxU_1hv~OZF3>0I4zqSoJNfatox-w|WWL}C*eRLmsSk#=c%4(;w=;+_Vk_)^lS13FZW{`>osI2ni090eHl7Ncjr&z zt^FaSuiE8`dd_f|8Hs)Tk4fLyD8C6$(uqmT#)uMH%Q=m2_EnnIcTg4=6>MkYziZO- zO6Azo3bW{_SXTIKP)A&J$5@1$2Z%CLzp?wyE6~SpycF5H8;JQ~R-$tqO3ab10&#}! zN#3zyW9DMmZT|U=DfE*k-s~c+DSXH5S3;S)dOYm0v$%Mt9PQpbnXxkSVrPuJWt|p@ z*-h@ctnCydTJF|ddi#hc{W!vxO?4RKJ8$eVf9$r2Klj9MS}-ucz7AT*lMS?I6z#i( zlXNGsuS+}l>yj_?+Sw*{lJ6Ae?`jiT`?{t@@Z*h=1}zy$W%^#hx2=yD{&0fiNn;c9 z*!cPc|3e7mpTr1?{#z?qOUX!-9`;JS*0eG8qtb$bNu82RdnpT#jxlCIm9)Ug`5jY{ zV<}kt=Z>VyCQ+gw+aWo#u~d+}=``b!=qs^vUm`JE$cSTC+6w;hFH25ZNAO#0beTOq zl<0yU%}x=oq=QXcultv@2^IEciyF1-*t+eF{Oi|qX@R$jc<6@*8wX?9S1P1v%QIhL z!IpLWxIb6uYI~ z@>Jo6i95ba2KY5zjk{9Hiy{aqsRC@G_5J#czeCBqMG?_=d&1v4cx zgJ|&=M*$mlKs@vI#3WJWdWLsnF@5?AlC1k~z-XTV;$uFe%zq1hFeS|q;?1?K!iJ9X zl97>7+RZYW;Ur2+%5E$WWEx2^2jVOlNkA53)c=T?q9-G1tW6PbRPPYOmQM@`8fdH3 zN8;Dlc8Krj%ZO*!-JxgwwvzaS81duOYUnyvnYDX-O*CtD8NYhpT;atR6Ikh-hVbSR zeYPwpPjt)lB>gi`L9E$b%3gkTO0@7{i#heHU9@JvlP z>B8}gbmr|l{Qu|!^y3`B^I1HHKWoH|9xx4LYp*8oE)+BhJ%?}6ryqFWvL(65sl^4< zwdI4{W*tJbPlouOa~P`IB@mMPHYj+HJCSf*6KPlvqTZ6*;9}!!v}IrgrV!YGJvpg> zj;1xCJtt$~y~IrXwqyf%W56P9yjhb3&A$Jo>X&+>PRnc5vV;nF z_&_{zHy$MsPEz&qnUo)KinOs&;AXFNL)%B%QPAP@z!^ zgEwb$UvzjjLwn&f*tU@0i9Pq`@8Kc*6AIGElxro*3%U1Ig$5OmpPrph*q~jssU>D8vQp`}~VI zR%1i>^olqG=5L9ml42~L4Z~8a_rYi3i{RVA7);acBC$_*95;I{0{u_5;k!dKF#XVL zAn~;#3`xC&fB6suMJb2Cg!ve@M|Q)h!X!MGBZXxP6+m6`OAxwVfKP26#(j25qZ#ww zf%>&?@Ll#<#9Pb{^mN@Nx(}+M-@ArEd%}F;y74qTXi_y$O#2Ls#zVklZ!^ffS_zw7 zB7ywckHj%Bik9Y{z&1%KpiJLG&@qj{?oE~utTELnMjWagP{ks;o>v*iA;$LF!DGH{h#Y`3iZq^dU4aX8*N+S`m9XOTiabBnu!QzZx@Kbph`slxz+lr}Cxi`vC zn^ZpeJYNIp6}Y22APKEksiZ89Penhrs*o3K`_Wy+K=?1Ag^VPUSzHMaKiCWs{e|fM%9})gcqytn@eeK8PymVb zX{g!!5|-R|374k2z@xn_2(9!$-}ZW8hO?vb9NGD>E;A9|d5MP`xaJOiF}wI|5K>NqpQ8bym$&_+ozs zvFlk9m@`EJ+bR?>r|2Wdb}k2HZ2AsE_m!f%xDZU)I~5t1zlPDLL*SCqa-^sej8uAi z@vY1D0G&BBvdoSFBrhht_7OX z7tnpz7i6-oHM%mf|G(eZ$2~Y%nmpQ}!PQx_4Md%YM+c17P;=&bQq0rET)!_;RCj?Q zcj!qzI{N7~x;rC`#HZ4N_7<+=4K%hud$qu-kRkQe+IK*=?Nw1#GyIKP32(wf=xhWWg}dD z>nm6lG9TWXtOmUT8bNIPTr_>6jUa36I)32M7cg5z2AuVif*Q77K&oLD_%wVPmq}QN zzN9&ze<6p!vDa?I+q_gT`gcDhWiBD^`@MKg*bT_$$RIjilQ1l>Lpu-Zz-X!u>XrQk zg!W{De*XxIcM}7L7f~>Cr4XCvy&iT2iqL?@Iq;l)2bIkbcxf~T#DwjG#mD87APCMJU|tb{5s1P=mWg*TAQb2}G>UFv=*d=S*9b1iiW{fZ&T6 zR{1>*>HXG4??SEM);pU3u9^tY)K2(T_c+I4Oqx7m%}4$%cIfNX6KIk9V%U@(V9rj)PDRlVy#J*u)L9g6ZudirE4)W3+-+S<%D*(^_v z94(=iozW%TobHgR!Nb&t9XrU#$;D)-poM$+YCiY-N}9T|^EnxH(U?p+`iMMfD#w-L zy`moMZQ$nBsBwG0{6>PlXv(fMkg}Mfg?JvTsqhO%l;iIR%;`S`RJLO>F%({=P~WUGj!YHW;9>SOEQsWx=nZEx<&54H#02LdTZR#Rq3a!snLP;Xmta zj!f}w;FNHcGdjndSlrSK3Nznx?0+VL9^3QS<`|bddFa73^w{fh+kwWPv4xhQd6-`Tj0+ zWY;3tyj&I;naqMq52T^vr)^-3+zd2j{UsDB--3L>Bop5Bkw%a6l>q+o51KrF0c9Ry z@X-?pzTv###9mxKarSN@F8;`!?=-uOIQIP(Hhc0V&^XB!PczK`vR~9;$@Fz##=p}* z*0BcvbJ7yiT9u1E)_(*t&!&Tl4eL0)_wR8&eBxc9-e_yUM?dBe`eq#{l}ocz!AN_C4%Ahr;V`& zTd?JFS10Zj4XB(Z$Jtrs1CI#pU>3aeVnY_FTPTH4`7`+B zo+l@?GJ?P=)a4L$Ypez00SiJfIep!Q^X&_WVf#^hs;dQ1T!M)(O@GKsmA`IFwaDNTR-h{L8ms#S}fHXTAyj)SrfyObJDw@H5zjqcX_8 zqYjUfdWvlw)FLcDYoJJD0$frn=PtRmj`C;>B@Hv@Q)$<^I$$oWPrH!a-+ zc^~FbVf(gH(nY%|wUL$NzcW50Z}<-SX6DSDH`AV*x4wZ~BuXMP!b9ME{xs5i8IQZQ z<_SeDiQsB<%5pzyZbhlF)2WK)IBGIF4?c8VqQ;dCDeo)(c%Ty(z4#YKxUQLxf0An= zjyQXBvPV^j^`pxP&NTr#@jxE7UQdN*jm)4&zX>RI9)p3NtH6`T3xM<6$wc>sYdAJ& z1;d!980T>z@q6`M+=*6(3oI9*CAVh7mOujKYc~?Ves_UhYY~)~uEpk2IH9XQKotHx zkIh^lg16>rg9z*-kXW1|mbP}{;r<$Mi+w1*T(Sw&KI9Rb&AtE)Z7(#SmrM*n7Q8sr zhO0%G5v%*G;r0urC}X`f1T(If4o@cujcHfFMo$|;Wf0)fxa4$XojZa332%`*CgR*7mtr zx_gI&=C~IGN@KZzcv#7{Hfs3cNWfP}ch# zUT`iJGq#(_(SNrVY8G4qf$r7#L4R$`w!xFw7Zd@u{&s_gYgd7#lWmB*8`csF*G&Ua zchlfe<*A%A(W8WvWET;4Zw~QS90k@keg#E4WRdD{ESeYa0MGZ=2J71sz_g($D86gU^UFRCUk>^?@iNXOyXX~k&MY%g?(8;0?9BS@R@zkMlL4@5u)!0{uf%$DhF(yJVd?5hH+~j%W~)FHB&t6sb(sT zQPe7vBh-A+elvNMKr=u8F3K_MJf&*Nr#i|txuv}_W|9L%+`{5LW{NSB%sw}jldCZw zZt!gMD;y;!XXWTY)PEqoF9bZW?LcLCA-T{z z94&adlsFXe5qznPK%5ijz_Xtr&_sCw`A%XD5%Ue?p56n;9+;A*`^q?~TbyBOe+YQN z*T$bIiqWAlcl3Q{9BdT(6K18KfL{e4k(CU-PvtLq(RL1DMfRxFY7*)8cK}RXbqXc& z`RH5dJ9x%ipWLnU8a)ct!TWa?fIZhLVDPpr$cI;i9xWD;tw-*lo~ON}M$vgrUM7!R z7WfSG5EE_}%kjt?xqdLCN#8JMjfDW^6uQ>pnQ-eg0B6Zwfg1k_U7GbmpnpDfBZrP!#$K)`P!`+Jg6e4Gic^!*-^jzh${{hiRIh=8Kr z6IgEEAsEuq3v9>;^y|)2xEKU@#S2Q>DhsQ zC~O9CHZ($ND$sS=TaJxtImoe50%f%n;T4;Y*1K(letS!Z<_b%6C22ZLzj~Xf9(E(| zB(~s&=GACP=Rd?0PsdJ}E+)gz8o;;CUw|qsg1w*miLD3Iaof^N+?y9miY6t&e;tm)c@zw>cK(t^cco!wJT!*MNCy)qTqa3_)1#nl@5&6;Gy>Uc_h`|DFekRY!(q61+WY20D@zjw*+z;bG4O$YJqHpu}_% z3vnJEl9`7745q*{ua*X!3*@0`1`g6wfOzz zBrZM!nO8S(V>4ImWzpC0%n{1Y+F&Y$e~kqESZ$dM9@ zU^1os2Gmggjgsrm6Gb1B@Bsq@vP8#|EC`TA>to}<_7%(Ea%V%dFBL~Sg3Mu~<8So+ zUl^Qn;4!w^yBZvw=Lp8syz!6ctl`655y+lzi4S%96K|iqgXt6NcpL5mDr8oJ44(x^ zSMr0i;g%oR=-vgVEk6w2$2yT<_X7CvQ5){Czydy9^%Y!kolV+!IFRxcs_<+;B@kRa z1O*aZ3#wo->x=fVq>p;U^rF%TU(OIGX2QU~Vek#kaYsM}5LX#KWZ)UTn_NS?b5 zy_r^zrsm5~8r|~R ztY{!k)y^VugR_K(;TaUGe+&kcpGEe~%IJ=-I%w1T4@f_aMn?+TL7xK`Ji3+-ia$(X zD!)K%NGlHweLa9Ve~cq88#JMvtqq`|=kNrd@E4_AdVp&sR3RU8O)|aB(e#?hYh)zq zAZA=S3|`Ygupu@d->&(RIA@WL89wKe6{iw_XN3s0UU~wzyX-=H3=g4WYu|#QIVE7+ zvKNrjPoZLM64W+~B4Rq#O{J)NM8#8Sv@t`Fv7 z8hezHtztSV+5QFM_J6r2vd>d0QAg9Fdu0gkOM2!6XEFSNwDJYLaY*A zo#1F@fWOD<;NmWOqu1%p(0C{TWqz*$PRc3J*km119oRus>#G3M`y0`*`M=@r{y>xP zb_&n^g26`iI+ipRy8 zg9dHja5Dy$1!$s>2gk9E4hMm%t36Pkk_8*92Z@lZZ@}uADbOk%h7QRLwtDLZ+N%Y_s8->V&U6>*@CM{S)_MJJS>xvAj)tC zv1^Ymvg}~dme?rB{4jtwa&tj+u`|k2fCbM9KzX+hfQ%{HrWrp&VcYh5uzW!yn)*5wee%|XTX+8kw^jyoe;Ij_i+>bT ztdBL!Z2LpLzId73p16@Jaa_y&^*n)^zXMUt!V%w|9BjF9^q=X3Yr z6`XNVhxGKk!S&cQPL=&FqspgCaq)yyXan;}Y?l!=GxR6n&pf5#-!I@=L_EM`W=6t& z`Q?~%xD;wQ)d3vIMaEQ2G58;?8!J;K6PW^zc(C7W1_LMi%Lkj>_kt zkGD2l@Y(_nG;ASfkB4v^V%|cmpGO3QD&T&rKcRlj{ite93aq^v4encwf=opt)YqYd zR~DO)*o$}+_b3s)>UxPjG~~jH?2G7M?{XyjpAD2GV71e5;|-SZ0~ZJONfX?)sgzm3qZ+C` zKb+h}#Zh;)($S5!wWMdxbn^J=MDBpaO$1i7AQ<$WRB*CFmbpFXWt3LwoULFYiTL3oyn}W(O+(LSRjc~t5BH?*? z5Ja_eV8os|q)>ewK33caCAF%s_Toyya@I?9#mWlGddQ#_^Q~~~m;<3#u^vQ5zXyH4 zWs#oTiurabO6ct%PHJB`MO-V}f#f(6;>bQNP&p+728>veCO#iwjQeE#o=Y?F&uJ3c z<9rTmnkS7;&AW%fr&|J%<{)_aZZ8qlH8KADf;<)`xd*!pBzbd3xqFqqBAtDuRPW_Eq;ujE z(x+NTnL3>#cYiA)p9g&7($`zK#jCBUuTC;l*!Jb*z2h>}>73Wpx@~W$XCBYF*#(B& zp1YS&QG6_lF^P(FkWXxD(2M1Aw!m zBepp?#%2BMKrW)-p(Ob ze@uaO7tdqnJHv_W6=AS}I}4nBG>!1Oq6VV9>mc)dfa5ZG2tV@Z0wj0;#Ev!9BgY@g zpy9U~FbtXl-@D|49Y?Pqy$4s&HtPpOYJW2Lcx4job9oGXdh$^7>$s}Feom~r)#UZ=^%MFp#g#H#j8445NP`_TNxhM# zR^4;wo=C{2sxK!{LWP&qqW72JmIftC|Kc06!r(WYHRwpyemur~IbeaU+C&kr*X~w}aAU$EyixlE zgfpE`$}$Cx$MR%!|L`2(ad#u(zn+1+?Uv#P#;qU>wj#=^Wyo{Bal|{#GpK264Z1Kb z9gcnUMbiIi&aW?+Lb#lcK{v0k6DFM#zL;J5Sgl_3wpH z;u2xH_{dtqDSsE9HoXZm9N|JO2MP(@)^aXizE9*YpAWa@E+r1TNr88#4iN@hi(%vR za?YX3pTzsqQm}Yw6j+v%iipRF@GAm?!OCk-O}gQjZLDY zP8pMtdu6G_LMJk1ogt~dDF!`Okl`jq*u$*&N^(!(Q*xm95}8zELK3v63Lzbjx{c*-abWNc!sV9_JyqXfnXrP$>LehhCiOhd_ns~8d zGNo90hRiRn;p9Ct0C{fjiLgIAuxIyWiO$R_Q~MYed{8-xjnQw2mDVO;O+qc9TmBa$ zMZE%-Cvbn@oGQ-8OAm-WiWMLyMhu4!nFI1)HeL+Q5K8sNKtvB2 zc2WqI)3FQ~2h7D+CSJh)Z97LOt?`FB+Mn@RW!WHrxh{snsXZ%r~Xr+hI*i?dBNTc{BjOm=uM^f0rYsnZvoMsYH&&p2J$k zFhCvmg5RI3;NSFcnEdNG9?j*G^cF)Px8e)(gBQ{IcV|(>x_UG;^*z`lzXZ%)69wh# z649o0!|-!eA|BPQOw@Y`LFMdFsNxcdCePjqw`KLCmaCtzou>DJt8pp7RB|wk?14){ z%8^U=5@hMFPkE`{M`Th05vgiMDc(1R`{P~U_y-60CNd8N*hr(I3H=_0pCdD8(PYy5 zMza0-DJZ2;1{tYHGSamidAH7iMI}7AVtWxvSgcBRP3|V2huD#Y{<382lR1ct&IiJ{ zRY>#87RU>H4C*S)$^K7C=#RSr=KM*4EGxDLd-nxGm9_tfvU@G2^L0N0zgHA^9b}HS z_|qJxL;I2K(kDneelzUvMd*+H5az0@0+P?a0rmQ_WOlYTx?=f*Q=CiS8PsOjH|;qF zwr4{3)?(-|_+f%$9VZUUWnodmM3h!`*EG3X2WT$Y3TxITVt&7)P^@1k5!!ncxQ$MR z{hMbKPbXEuQmON}Z^JpX*jx!5wx>{saoB%g} zUx$5|^_93f>j79>fpE83X@twc2*UeY2Oxc3V|BBBgACUp(CFez*v=%;l*6e&-}D{P zHYo}W6)zxOyILW#bTTplDa6^f5wJFK35aQ317AMvLuE(aK-KtKxV+g9d86iDxZO^WF|VjrvS~1jWN5jb_REC#YyUt$vU%J`zom3JY?3tSA()+mCcrf z@8X*I%9*Lj7@5siImVS@BF*514C>C&ddi;JWR|y~-R$C`e9Etlt%nlW4;^GC7VC?b+;?dnW zu=(gHwz5+LyUQIX^xZpfjmkw}+D`+#zi5E)_1FyQH#t!I8wFM@iN{wy)4+qqb>J=g z5aMEFEC~8?84OxI$HKYi2!$_3XzTtk;(b*YsF6#DS;;DR%M)w3n&~CFX2)SW?N#8x zJ1?Qjw{7@0g_ERGeKbsZ{SXF4+5k#c0c>oUjdtHE1gg|Ja`ve*5Ioy}{IRj0B@6aImP6+pEw2%S(1!{oG- z(8!Z5Xj54h2<0(k@~tbx*nD?rUVa-&Y_gzi7?0#V6p$J1M?hg?D)@V|5jMW_M_bY* zC?Lq4Jg3}@q=SRWUoB5e6HgAJ*{K^rv@iy@UGT^BweQ4PH!BBrC>EM2J22ewxU19= zCk5^r*$Y(bh7>CMLIHW=LFom(HWmRl572-m-i z=C0{S+#P>d>~4(?TAJ8JXde6k!n?D3N5Ef_0GhjW zpjEdtavD}AA|-slJy!y%L%TrY^$S3e^~tx0!+#O7iIVp9MIv^eZ~vVhBeMr^Aeq3ZQ>1hq!;z8uFhU2M_9{ z$TJyX;Nz=AG`w#Wc|5lm%yKC~Rv#9CD_4)>mHS&z^j1sq)2pvAMDYtzcWJ^$WNLs! zG7&Is#0GAP?L(*Z8SJoTAyKqe3g#-jhd-jLvBS4zi1{5UaBM>{CJHYFb4LIg9d8D> z`3hp6Y9`l+t^X6)EME+dE;x$jJ{v*p5>v7+@huY8YLVHWFL8ouexQSwmB5h|nfOi1NMiGk zv#6IyfN_DIT)p>p)ZgYI@5h-~Hmj zirW{Fe8yxlZk;NDYiuAxUOXV{Rc>G=#?8dZPzWx$QPAKx0G-K>*oJ#0FdrKvHnr!2 zw!kdl@o*(^XT1yfd{79r^p)VsscnSOvGqhPTn8V@mqT|Z68{&!0LTT}fuM*Jgm{Yv zLYD8L>(fyDGB=B;@cBZ#K5PgZ*4}^{p*!B9?gx{q*CM4+V>~u66Wz|4@Q@h`z^0qS zV3VON;gMMhc^)J|zbxz&m&MyC;~?HUD+Yn1R6PA2Hr zn+|Zwkiw5OJq0ce1S;583-3jBayPW@rSxo5$x6qY+^{Yw%7nhc&3_wC+PXS&ogaRp zZWkWlzR5`Bn(ewu&Gl9_JLfh;mLvlz>SP7i^MH+6S7A4K#m$2H^;L#aPHdpkLwIIG zRyo}FoJZWD?z`MsReI!R-^pCR|7*Jruqcvl3rHA7;t&OtAQGm#tE#&P7>1yzAR+=v zHjM)ef+Ph*L03T}tVl3|n81LTT~sod<07JB&LXD8ydo;%>w#r|)2eU(_xL_&s;lm; zIk!)pdwaTj=Gi-{?RF7*9%n_Lv{2w>A56l>#d4rGnMov6rGWapDR|bblia*Gb*Sxq z6Aw064)|$H@psB=frrK+`1=fH!vz5o4*s>^q&AP?(Kt9gNax>!4j)T~VLs~GR&mhz=XcaN?Y$Q4L z?SWF`S)%BT}cPi z)~-j~^fP45mOOOKsR3V|c@J>UrJ%KGX~gKNhcMH6D7kIuX)pv0fELv@Ft>IVnH0B* zJNs!R956r^Tum0>i*$|2=6gP<(_j^}AD;u}20aB{wTfs&c_}Wt$cxlIlZ`5>kD^h# z3ea<2Gd$C94rLwjLX$*t$TLosTvJkpvd&%vcZzdC=b&|PUm!plhZZ2shE}qqI1f#H z#iC}J`&%}@vn3~0=|Wq@Hq3ed7Ln$d|)X$P@S7 z(T#YVJV%_M2VE_Mn{G`bho{^}3JwP3k$Wm+@JT;%_px;d=B}VSwe{%N{C6aZOCj0u z9w_IC6}f>GLk`@kOj$fnr|(=JLMb2o07d-wBR6tn+AxPdwruqoT4e9gn$&`@! z?msR3w@5`q*pN`>h5@E?L<-zoqYM=yqVXrICNtY*^f;43FJd2i4VwkmpQ> z;TM)4f(o;hpmS^+ciFH!csKJI@!^#^ypU}{c-`HDd#!CCqS9gr7|;l-vuubMBX9Ud zKqGU%VyrQ9BcxK-64ApEC&7x?9G)i;gdsz?HM&_GTIkUN}UPD^AACH z`J;G2at;3Og%4p+>H^b;^syYgDF`2Kc>!-XK8o$nXeRo3KzQKyF2ZG=K6kZoIpLFP z4Vh2=vE;jNCw6uXhD+Nt34cv7YCA?@eGSWrHwWdgGLOs4czl|7q zFc}=;%z?WN)HQ-dD4QZvFgP+{8r>Eeb_l%<~~t+qXaysQvG zE*sKFotQ1AcW!@z`fki52QM=rjUNx7+MnmrhR6EQCWcz{OGk`O(J(|IeMA(NGM;*@ zTnZyM)l;&e3u*ml99%vml{gqXhFEBw4x&!Ff%v)%%czOkplP2raY_(Nblx%t8*8Kq zx3g+Qj_)?`de1VDqppE@=l12A2D}GYMPE>OvILjzFaWtZdW6onUkLe2Z(-mXIoP~S zmuOG{K(?)hsCvPI&H?$*wzLx)KY`0!Z+!(v2v-towyKc7u#2H$%P=&i@jW~?eiw1+ z)=p5`ZVV0QUxf4DFT(g@Eu@;AOj0|Xz}1-PNDveNcDAnq;p&&*xcn?~z$t+K&{%{U z4wM2T+ntfONjzA!@ILH_(jZwD0dPA$7)}o;0;XUwsS)9Y70qx!2hX^Ia_(--;AIp! zb;KC7G_?j^*gX(V*S-kW!$&Z)gNw7L?;w@67NUr94f1`ZIrCnfC#p?pN3m{BXmf}$ zI=!Lmvq4=$ryrRQMXo^)Y}Io~`7>=VQ@?Lm0Jvhz+h#tW5HD>3{>; zwcLwml+d$N(y(>oF&O#5)QVqwk{;Z|rk}l6qjM)7p?oa->2{|;Ds)5%uc>1m)#kp7 zwl3&T-%{#N`@gEh5HL+?>$AMB`fCk{QKkm8+Vt}7V_ zRPYL8A@6#ho#b_`W%Qi2%V<8Q5=D#F(+}^J(lU$2;rR#hNUX{PgEN&;Ez1|m^HuS3 zr+q}-pKO@rz#|9G1+aPdEp%+TK6ycLGvJxB$ODG%M0(8uqHD^2LU3{liW!-Q?!R4* z&(c{B9`5AB3R)dLx}-_U6wg69m?7G^nFCX`i$I3*Xi^8ii)o`&sBmxrG4!`gt}w|G zU2JpUUR*1SKWh#F8cKOs^Q>_Y?0sd?Hu?n=+-}8W4jzZMIHu^gOJ1-az7#Iqp+yKh zw&SfjaWLGw5sp!L3(gg&;Whr*1br%)5RHEfaw5|~;k68KABx)@x_q4LJsTyP@mXTdKOdg8i3#K+DYuqT7Xer>+xK|8kc=q0^M(V7>VeLMNye3wU~#Yb&<~Txr-q6u+s!bD#>i_~&Rm*EE6Hx8fRU z<@uv#7IJ9UwYoHlAxDu(kj z-0?v|U)-my0Nv$;NyhVtV_g#&_p7jPZM`{WB};w zyADsPk+t+z97XI~xC8{ScMy^J`Q&|ZA*@-Hgtoso!T;EA392{0hpZ?aI7`QpT)y1| z45^bMFEw?LDHgO0zfA=-9`cw-lJl;&+_Af@bbQrF58H2EP5t{7SL=3Nq zMyG2%$z7vTaVt#)`18OgcqkGBmT3CS&Sj3Q;58jba$5Z5=T5n0|5_N38^=2~G&<`G_j^S_5g);XpI-m`4 z6Uje5)Q~m0eyC~l{kA4&wWxZ@;%U>itv0%7Peld>TYNAM!21)b5y3$9{ujp_c>fr zYATn`Wj&;3V)-c8DUi;6HJw+ao{PWwScq;!j|SJU;i$c6I21ikv54Nf9-6$8gL@jS z5PQ%hXu+KKTfNx~F0@Od`2h!zl1>znIrSB;60{%sQyYnUcDgW2+8+8n^#|mkTI|vx zB{KilPSmUNPckG0%z^UP@!0w^@k~r(EFTWF=TUKnk}`wILSo ze*q6H`je}`odg_LZ^5Sx-GcWOW#FxZ5YOLM#GKJJ;r@Mp$7e?CfPP2@?8@DNt7^}{ z+S=q{%wtIvEvP}ajqheNvOkh%OIwIz?v|A@)oQ<>u0Ir z%EMM&Tn1dm&BtQWRw4Pl`-y#jE+pot-vni&&*SspVOXTO6t|dFhQ^JoCR)?xp^rQN zMCdT0^xKpvfPQbg$`-r!KOEm&~? zf$iJ0pk2N_Do+>>2dhtk&&wMjyuF)8>QWiXY*2*$2RQy%N=| z(t%tmv*1DvQ)*IBE*xf)NtGU1h)N;j2{fM$crh=6GL1$3>MGST+dj8|G%gc9DUT0!QNWPH7TBcV!4JE)SG6 zJ>e3+0_eB14xf#e;M;g!c(LX-aOz_h*mbHDBt%aq7VpWy#4S?93K0p*-*|#eZi9&X zA0`mdVn?h?l>~m1wqlWEyfNjze6Zkk5%f$ui9OVt1|JzN!Ly#Ff~xEF(9NgavTWoK zsPoi-u&6UfyQK1nyEa>3$~IegdHzwDe9aVAw#yL198VJRUbkW4&^(}K(N1h0*GMeL zl4Y*bxMGBnA{un>7hFqeCvLKEC$xFgN?dqdKww7ts90(^ez3g-TD7^N6E?Nnmw^LO zzH9`MLOmd|*X;r`yzgN7Pb#s=&k(m&Jqv|&o+BDwKY^XIH-N48OX2b42{19y7!2KH z0PY89iW<%}j5ea?|h<%Oh?c@#CEb^$#?z90Qk zD~HY$45t+rsgMIRDk#CcdDM*(0Xi19n7W$dNGq(qj8_S@(P@PQTz``<@IIXcbXH|> zrIiBk)Van4)wTehJ2R2+-);ssIR6Pv4a>o?rE+j&O$Pq%DF^%IZVH|}42Sn>6W~fW zS?v6EdBQutk|=rRg5%k~@Jc`zn1-bSd9N1)`_&A*vsnr0tZ%h++o2BX=on0a^BecZ z{(5AI1rf2{8Q8^_EVNK_B|d>}fd@x?;67FCM^>14gK*DX=o$A9q`A=!v7aWOlW{Dp zY4sjrx=|y_i77(iH%nlUe<;}Uq8(65T)4(!(`kH(c_eh0ugdKsuwt(58h{f` z&2a6hMQCpobDg$J0jez>hCe+ek2#Abph=IHqkdBe*r~r1xQpVz;SWVn?f~?E=J277(NCClH2p_ z;aSX*%2n1udTX8_i~E~N-l&b_(2`eZn1M7}BLAFdIH2xmoSgOiBVb>Sz8QGuk5LAF)R-A#hJ2Ch`>L7m1 zeVrwKIhF8G$^{+zH{gpyGVrbcY~oDW0&L=)RIv4|0^F6cflv!u4ZP=F#V?Om#M4)% zfseoL2W^(E+(MZ|Y_N_m+-1tL73w6`>|Ll>eKa|_I>FIg(4XjErCK;wa9!7}GHZFV zUD;aQD*0u;HCt{e*@03uJAF5KyT(AiZR>lRYBM>ln!NRQt2<7e73i7eSGNh>Y*zFS zt}d<^QWLt{RdD*ij_RXx0&6I{LR%4Ey86~{QPrZ1$g0d!`vvPSFRO8_n9pBpoKbDI zdSunR&gWHuX^X0C+!OhyJ6~5_ym7PgWx*o>s`n``TX(<8H1-`o@!Imr5Rb4*bYXhc zvt7@swmT;B=bQKl`bSKydR;i8x-_=F%IRYz|H^dznnkX+_%|ur>fQQpsvlMB+rr~5 z)ssC3S2sLeU%6$|K*5qw^{NSIu9gSw8gVGh35W)8^W&OK)rQT<$vDEa+E1K<-TKq=zPTH$}&5 z0}P3}cAUN?1iy+ahehQI2O zTg{3&ysA6d8P!vpkl?J!WPSsU39=H5s@A{w#ctJw)m70__BDaNq}@DSOFJX|6M~ne zJNVAyeyv^`(zixiTdMj;aANhoOSN#~Y zr=8~Q-8c$4hb8mE;@{HG%H?@)_dTN98(yH78*Hct`7h|y$}D=;?M(W}iw>$^^KzIY zT13Tny7NZwUqd~5Z$iJ)n$HWA!%5eu3X*CmptP>7KoM(Z(<6%B@bnL0a8AQfN>r49 z8jaVId0`JxnZ^dF7O;UlpqL3a1ni=_!X2SsNglalX)>BIMhE#97$E7O^{CQq6)C+- zhCH9Mk$PCWjJYQ@kV?Ngj69r=p}H7(o{7vuWcb9EqWWy6BXmZfIltx7!kaFXY=kD9 zZZ?XVgdIkQE9-c>TUOZ)I2_K;OE9uYyvVh+o^aJ>KPG2u<}%#IicYp|oTbO_XlC0s z78}^=oJ!%Fb~@PRJy~QkF@1-PLeo-POP?R&3i%cH$*8$)y9_M>8+Yvn%l#OnmLZ&M1z ztJg)8U@ld3K^4t1JjwGbR3qi*IP=CDI`bM^X`Z@UChayk1$B%$P5H{6r~UT6CR<`x z(reUR>9?C2$QOyW)EK7%-tgp^bkzEr^h2u$yzJ@=DCE8cZ8hGR@>gDjSgx1o>+6s5 zw!FDvS9q zE4yPq@yP8Ou}@H~SGi2>Y3_v@kIOS_Ei^aOI@~F?QY8A=G+vM3g^j#uJ$18_RjpzK zFMzk(3VobP7YmYYcBW)oC9iU~$-j2U`T{OvvtwX`bsxNy*SSR5COyv4x<6N!_hhcQ zjh*HYo4NI+R$eW~tRD71Y5n%zZL31-U~5pAVho7EqI+147Jl~xh*Gp(ZKj#+CR z*RVksPCLr)OSD_7er$-8l#!EE_m7d&0ZY!7;%)3g8MZD z1cTP+7_|QhgU&x=(ES`kzt1q}{WJzPOP?dp9NTDdB=f=zII=xfZbUylP=+hT-Yvt{XBlyHq8WO9i-v5LF-N|e;&BX* zO*jfs;`rF`;5fD+YamCzXH|TgrRpqGj&jd{UYq{|?tf19?-KZbvl7ghRb%qys`+}w zbjR143B(|d0u!xZu}~rcX1xu`QaA5yP)x(3w?Q+Zme^VL?vU7Vdm9ovthXVt!+RSN zJIP^6k!HwCaM+&+genVg`U=D2B`jxFbsU?+f*ciLq*xRi9UeV}SqJfAwkiur21SR6 zX9+{XgX0-5B;zG2CQ2L~KO-uR89*@u62y@)!HhUC;-bl-IcmZg(?YseNxXQPa8{(K z8@CQygT>>h#))GSB>w-B4d%u5W7zjKGrMiR(SMC8py$uOz9C{lP(m3`uddW7pcZbAo zNN+=8=hWMf*bVJ%^pdSbB8kI3m&D=!B#F-dEQv0h&$85r&m__HrzKH~HS$}suEKKr zTg;gZ#Sp3g-^!`UNTgbl*-+F5JiJnUM$LWzziKa60Np@5ATl*wqmr9g! z)c5%5{s})vf67mfAMw-kYkvOMDM^X-Z*!0tOYf)osmc(@tCv8C36o^Um``_1;P|ms z{G7&)bQr zs}A(o`t%euzC8tvUr#~f&lEJwkN}Rf6|)KCNLw?TaU5wIW;326&1W_fIMTLkdDcXZ z42inw@GbC)f9FU&^mjcAKgpaw0zZ!9^HoHZKg+Xv^e2u#>-mvyNuVSJ8xsME3Ng?n zl^ugwQrR(JC6yh6@;{Xw6C=q$CODFTOjINTnQ%x3PV61{W%&Ctof?c`cZ|O5{9gMz z`TILv5|7<-Iq7?HDf~n(Cx0rJq94g+(ARSLU#BZ2*1ye9W-QmAmdo$O`pZ5(lZAsn z&BDs8kWVwJB1_DXi;syHMY0uHp&Ys3m>JRW-ACA!v7GX`W!UGI;s3Fm`j;hB8b&as zVQ924Vx}l|O3z8h6prXC>KizPh@xPR41TSJfPKeG5k)%xd&61wO3*@=FrWYd`k zf4@veX5U8il?;Cy8~vj-{?Ai>h3zM&MrQTzr}+xmcc!ua z(KM4d0bilf_zTTvmBhDU*!+9{3GgTU^MB<3TP2YCXRrS|(XsuJ_lX?;ucFiAz4z63 zd#Ap=Fn?IEd?`zkwxmWD_avgw-n6grgK8%V8OrMA7F~7ZaMoA;BO`Q n9`*aS({5q>ky8nz@E%B0KZW$Ej{XZsp6ym1bI4zhPVdnhdBm$ zf=vr^^a*w4U<4X72WSk7>4h)l{2~hcCHW@CE<^7br zEGi>qOjw@+)P*a7`hWrK?z-tVP@@S68kxWv6S>#U5|t4(#jlYSW*f6c1B>bZwOej7 z7>XjrCL(anh%;y{*dVbLH>yNsB+T*afu|&e2^)dh7)_J7I-7vnEQr>&39RkLNv&0) zGQyVlwLzkB0yr9VZEK%1s7WAvM<59xMQ95+K5rkJ$iXEeNkoc+#%BvJI9ArY%$Ug~ zBT69sKx3t)7ibrYY2LYaObkGf%FMt3W*nR#CIu2;1Q7>5E!@Ds$Xw3Q&IK3a(|;q? W!^JU^iytI}AB6vI=vFCpzYPH1UtRG4 literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 382941d9..299337cd 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -84,26 +84,12 @@ public class ModelLoadTest [TestMethod] public void LSTMLoad() { - var inputs = np.random.randn(10, 5, 3); - var outputs = np.random.randn(10, 1); - var model = keras.Sequential(); - model.add(keras.Input(shape: (5, 3))); - var lstm = keras.layers.LSTM(32); - - model.add(lstm); - - model.add(keras.layers.Dense(1, keras.activations.Sigmoid)); - - model.compile(optimizer: keras.optimizers.Adam(), - loss: keras.losses.MeanSquaredError(), - new[] { "accuracy" }); - - var result = model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 3, workers: 16, use_multiprocessing: true); - - model.save("LSTM_Random"); - - var model_loaded = keras.models.load_model("LSTM_Random"); - model_loaded.summary(); + var model = tf.keras.models.load_model(@"Assets/lstm_from_sequential"); + model.summary(); + model.compile(tf.keras.optimizers.Adam(), tf.keras.losses.MeanSquaredError(), new string[] { "accuracy" }); + var inputs = tf.random.normal(shape: (10, 5, 3)); + var outputs = tf.random.normal(shape: (10, 1)); + model.fit(inputs.numpy(), outputs.numpy(), batch_size: 10, epochs: 5, workers: 16, use_multiprocessing: true); } [Ignore] diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index 58c176e8..3910eba1 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -65,6 +65,22 @@ PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + From 7cd829288de2f04b701ff03d29edb25a4d151844 Mon Sep 17 00:00:00 2001 From: dogvane Date: Wed, 12 Jul 2023 16:58:25 +0800 Subject: [PATCH 54/77] fix per_image_standardization run bug --- src/TensorFlowNET.Core/Operations/image_ops_impl.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs index 0ced407a..318b8b14 100644 --- a/src/TensorFlowNET.Core/Operations/image_ops_impl.cs +++ b/src/TensorFlowNET.Core/Operations/image_ops_impl.cs @@ -102,11 +102,12 @@ namespace Tensorflow { throw new ValueError("\'image\' must be fully defined."); } - for (int x = 1; x < 4; x++) + var dims = image_shape["-3:"]; + foreach (var dim in dims.dims) { - if (image_shape.dims[x] == 0) + if (dim == 0) { - throw new ValueError(String.Format("inner 3 dims of \'image.shape\' must be > 0: {0}", image_shape)); + throw new ValueError("inner 3 dimensions of \'image\' must be > 0: " + image_shape); } } @@ -965,9 +966,9 @@ new_height, new_width"); if (Array.Exists(new[] { dtypes.float16, dtypes.float32 }, orig_dtype => orig_dtype == orig_dtype)) image = convert_image_dtype(image, dtypes.float32); - var num_pixels_ = array_ops.shape(image).dims; - num_pixels_ = num_pixels_.Skip(num_pixels_.Length - 3).Take(num_pixels_.Length - (num_pixels_.Length - 3)).ToArray(); - Tensor num_pixels = math_ops.reduce_prod(new Tensor(num_pixels_)); + var x = image.shape["-3:"]; + var num_pixels = math_ops.reduce_prod(x); + Tensor image_mean = math_ops.reduce_mean(image, axis: new(-1, -2, -3), keepdims: true); var stddev = math_ops.reduce_std(image, axis: new(-1, -2, -3), keepdims: true); From 0cc25fbc35eb406c4f7e93ae9894633c03bfadae Mon Sep 17 00:00:00 2001 From: dogvane Date: Wed, 12 Jul 2023 17:00:16 +0800 Subject: [PATCH 55/77] =?UTF-8?q?Add=20a=20function=EF=BC=88get=5Fclassifi?= =?UTF-8?q?cation=5Fstatistics=EF=BC=89=20to=20count=20the=20number=20of?= =?UTF-8?q?=20label=20categories=20for=20the=20image=5Fdataset=5Ffrom=5Fdi?= =?UTF-8?q?rectory=20method.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...processing.image_dataset_from_directory.cs | 32 +++++++++++++++++++ ...eprocessing.paths_and_labels_to_dataset.cs | 1 + 2 files changed, 33 insertions(+) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs index f42d12cd..377ac4de 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs @@ -8,6 +8,37 @@ namespace Tensorflow.Keras { public static string[] WHITELIST_FORMATS = new[] { ".bmp", ".gif", ".jpeg", ".jpg", ".png" }; + ///

+ /// Function that calculates the classification statistics for a given array of classified data. + /// The function takes an array of classified data as input and returns a dictionary containing the count and percentage of each class in the input array. + /// This function can be used to analyze the distribution of classes in a dataset or to evaluate the performance of a classification model. + /// + /// + /// code from copilot + /// + /// + /// + Dictionary get_classification_statistics(int[] label_ids, string[] label_class_names) + { + var countDict = label_ids.GroupBy(x => x) + .ToDictionary(g => g.Key, g => g.Count()); + var totalCount = label_ids.Length; + var ratioDict = label_class_names.ToDictionary(name => name, + name => + (double)(countDict.ContainsKey(Array.IndexOf(label_class_names, name)) + ? countDict[Array.IndexOf(label_class_names, name)] : 0) + / totalCount); + + print("Classification statistics:"); + foreach (string labelName in label_class_names) + { + double ratio = ratioDict[labelName]; + print($"{labelName}: {ratio * 100:F2}%"); + } + + return ratioDict; + } + /// /// Generates a `tf.data.Dataset` from image files in a directory. /// https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image_dataset_from_directory @@ -53,6 +84,7 @@ namespace Tensorflow.Keras follow_links: follow_links); (image_paths, label_list) = keras.preprocessing.dataset_utils.get_training_or_validation_split(image_paths, label_list, validation_split, subset); + get_classification_statistics(label_list, class_name_list); var dataset = paths_and_labels_to_dataset(image_paths, image_size, num_channels, label_list, label_mode, class_name_list.Length, interpolation); if (shuffle) diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs index eaa762d8..232f81eb 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.paths_and_labels_to_dataset.cs @@ -9,6 +9,7 @@ namespace Tensorflow.Keras /// /// 图片路径转为数据处理用的dataset + /// 通常用于预测时读取图片 /// /// /// From 68772b2cbdeb431a432617e6a5e8bc5e2b2ed754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Thu, 13 Jul 2023 22:51:49 +0800 Subject: [PATCH 56/77] fix: use git add --renormalize to make model files binary --- .../lstm_from_sequential/fingerprint.pb | 2 +- .../lstm_from_sequential/saved_model.pb | Bin 755111 -> 755111 bytes .../variables/variables.data-00000-of-00001 | Bin 61038 -> 61038 bytes .../variables/variables.index | Bin 1373 -> 1373 bytes 4 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb index f6ea8da2..c37cc37b 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb +++ b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/fingerprint.pb @@ -1 +1 @@ -沦Ʉ%̟땐͉ Σ(Ћ܇}2 \ No newline at end of file +̟땐͉ Σ(ռ2 \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/saved_model.pb index 6fb7c3f0e8e4a35afa38cada60f78a6097b84501..618c800eb45f0481ad202d25bda12a86680eecc8 100644 GIT binary patch delta 2303 zcma)7O=w+380G!H`yL{e2#Sp_@!iJf&$Rd6`A@a6-B?W8DiRbPNt~HGvk;>7hc2}u z8X>eQL2fbV#zhwCq7q&Qx|N`4Hbq256c;YK%0}oy7drReSl@Ei_s)FhoH^%wTkA)+ z){kC8n;}Fr!dy_mq=*2LhyjL>2GS5sQXCTlarW_aWoc)$dE?b` z2r39LMj4hGf|QQzYf12RCA)hfSkCs3H_6Wa?%TsqDo&VW2x!hEKnyYQSC;L?waUyXubXoUQ`zsgTJszG zyMaCNZROUI7ny>4c7CQ^x7&m2pKE=hL>zM=fMAXciWmnXi8L@WcAz;$RAo1Lb26J- ztDLmGFRMSD4Vg>~L$L->=7eIdfFMu-tPMm7jS~^W{}E=xo3&u{VeQwR*TG!w!&Q6r zq1vT^4?3Hpt7ex_aLBHosiC(*A-DKBXGsKeE4_OH&gePO@8@_?5S#sJ!Xx{Y&vFIsnAb;162tnt-S$PAVQ3JxG7 z2|y-+Ksb&-K&*8tWNc_*g=O2n+<5Uwm@p0ri7|*{mo=n$9yk&KrKO8ZA_c?pfY9Du ztw-!1qsBzPpU1G6f+;3$ig1C}3Ijzn*Tm3-=+dN@k2RjPAN<@{p8e9B^NF zu63c;_vl)Ck#$`-S0{o;?c;B>u1|VFE6JCq7B<`|pB^X-ur>EXvzoc?IBDPAY2A;# z12<(|nSMX(CH0tVPKE8oBVKtMJ2dl5`)}d96{d@px7V(=x1aW6Grq2hPT0NOcAem{ zgiP59)@`~I5Bk}`neLNW@7Kv(JkMpl#cE(bUkw%?^QxS2J}wT2?AguWdg>LTT=S*T zd!5nCK5Ev|^|D)=Ek3v$>l7hMaR#{2!lN?+Faj;W+9uGZgkv#K%r(C) z{&r%@SeSB)A+SOw05hY3;3flxN+dAh!ese{gOxM+(l4c``)TQqW@yQzTW^&<+82b# zy~`TAi}}Ym+UZsnO8Bifk=*m}vMP=I%Fl0=zIZJh)E-d2 zbtLHb+o1k)%Au_Sah-GJ?vgLftP+v}q67rkLJ9;!#>++(5*`^-LxsQox%^5i1ex2> zDgRW+zqwdj%;)|Zn{=mo<>jY>3@`%H{nMzNoC>1$$ZzxR&x4g8+EZM6T@g%ymB!2M zeGG(5HAtA0BGt^|B>#TBc-rm#r1I)?OteNSQ_7n@^WGA~0H&A)Os5&;1}mcIaHd<2 z)t+(px|Q*%7+NVUC4M~xl>keb`kC_7zd>Kebf$GSg6zXPm1P>#>QT{gcWFnJ&4$vM za6Mk_wu6DSU3|6r`*T5{*y&EJMmKcO25mQOci$iDz8=PZV diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 b/test/TensorFlowNET.Keras.UnitTest/Assets/lstm_from_sequential/variables/variables.data-00000-of-00001 index 83f2a2fc81958c3a0916847f3ca7e7db48524b37..ea67db4f4886a56b01b78846fcbe2561cadb7691 100644 GIT binary patch literal 61038 zcmWh!_d^d}7p`b%XlZH@Nf|AbKKEP=v?Q`6BO=+pp$HA7lD3M{PE-^b)#siYN{Avt z8ObWEG73?=y??;{<=%6j^PJ~-j?KcAe6hhj`sn3*2(Y@r)iX1AZ+#_K+HAyemN!nY zpT<4?wRxjk1Y2Y;#Kp^^ct~{^{Y+XgXzn_C?nfQnyFDAWwBDvGzXsF87O{Mt=U<#C zUcs;3D}~+1i!msrkT$d!@zd=)`Lt`HRC&y8{y=RN&(*%f+fL}S>SL3kBc~Z(JQ_}a zJ{Uz?9uk|wjcfR4XFJ^VTOWG=9YJ05?U)v6%x@MHvNMasXqW#AZcsZGcB#%KFL{l? zs6vI0IZF81wkU{t7LQe$kI0r5Q7Z4EgpYU;`LkgHUzjPyPk2qBZb=LIUfDr5&ioGP zyZ!~dGUidE569Rd-TNMLHZ7r9M%CE8R*ag>8->s09O#(AnUGdDhF%`jhwh`TM1Qm* zUDPw1HjT3g26U@|4W9UE%Nj(d zlkr{)`B5oDi2vIL`sR(~$QpHi&nO+WAFbd~HPyKBpei3%;K-lf@56sDilFV3B^UlN zrs4N~v%8xuxn2)|!hKbqKWDix&0UG-?9}I)?a{cT{4PxFb*9ci8YJ!QDD?e#6oUVZ zgURQkVWfH+o^Cmhw@&up-yd6{ika{|r*ybcXD_q+{2Rn$b!oGM3a39MXkL{yX{gnK zSLz%;BtJl}yDuOw;2VzJcTsqcf5NwwS%MVvf5K2zEjs7vB>H<*9s1=-v;4DT&?}%8 z?(SO3uSu_=u~tc}eXkY2ygUf}riL)NE<^sR%$Wb)<<1X;>T^3w%6*hYL3dFr{Ce4s zOHA+M(~PNf_1*`dBIt&X7CkUO*P31p86Zo#6hXlw0kO`t>^FDS@@`l|`4*@&u(Q~aepBgGc7afVfZ_ab5vuYC*E7{Tk!48}kxB-7X zd2TcHh%(jc+=T~B$I=qi<(;0YL_bBGzEhB)_cfZxU5x-NIdmO2<^9H^D`)a!bE^b( z-SfHc*927h>Pg4XH0SNN{V}I~EcF_HiIq89VvMXc6U2YS8_UkZKS^~8!85?jZYsyi z9`yZdYZhEQ4^>>WXjk=DY@c`!&}0^d&*;U@RzmK4FyoRF6zKB2c|3ZGIhPO}!EY)> z;7w``a;ZSQRQlk^*wH*zz6o1%KV$bEL#$W334T4Q{H~83UA}7&WM@2pW$|+GGpY;v z4_R_|>p>W_GNe}@%W$KCEEMURMHh*A(D9NQyeC?hR@kcWk{8Z6uiS>+J7R?YEt^7z z$$Ep~o^rvgMmwV6U`t=z*~07}+=PWXE$A<%NkdE~@Y?m#u+-oJ%&F}s5vNpXhC?u( zx-Lee?u+r@`~rcCPYjI9FhzcDH1|2?%@6Lkgzy|d3z^ewMfhd-Gp7hTP#H_7Nz&zq zFT==)NwEISbUO9TS8}KG9_%Em$>b|bu|#wg@BZ@~PYg=Zg-a92+nLt1WwbGzA@6ZS zr6KxTOrgJyyV9|$llkAqzhL>g120&H13Vyfs*eaYJgi7>p7ns6>N8oEr54Rp3&gWI zk!;}yQx;x2gSNHTK(GFM`k^vac;85dUOh0JZluPPPAP+#62>&+;7&At-HLNJjOAsE zHE0EM;+EWwnpZU8qvL9PW71YktjdL@GOvV>hW^9j%GIcuodo^+MbJLTf=={u=PHKc zT>Vf3$QDGy3l9UVU$hy6&L*Sp##}HkMsQEa26M$>_)0Z~46;u0^S=!IB|Dxj%3cmu z)BeL+i8##Lzl0j)R0<38Q`psi+1TcC3s#xEft3R*NPy}Lx-2*tODt}{>TTbJh98~j zOX~z|A2@>kd=Thvo3buMd5fwGqi~d;I?A5jdWkiXOA{=%fQriQhF7I5xEyKWTh~Y(Dbhl4k#G#jq6oGtoy0(go(efWaK2+GiwE(H{v8lkN$?q<9p#kMuNaj`~(~n zdy4brzr*�FaG03kk_kaT%79SA{qq^V0gn*-U|@-X}%A0u*0nhLCd|F1yu{ou3Lp?!lm7mV+D(6%41d&rZj(OZw<5 zF$+#lu){^x7unoj?fByHd**wx2^PBPv*9Pgq3e$p_N+*Pn{PhC%zqc4x!3cU{`dO$^R&qN1rPY%P2QY#^GjrH=)%+A6ln)3pR-# z0M9wjVDE%rRiH!O^^GDQQe*JNVt~SLnIKJdP-(d`4A{Ei^xinQt@<5C?vjK_ffG>8 zTZRRu&)|EXyF-7<8wOvBU|ab}G`*pXQ>iwc8ZU}d65g@f`RCAQa2LP(Z5%4vd|}U7tWKP!9iwRI6$o@LB0(0h31`60UQbr&2yiJ*N+1nMP^lP+IBzWLQCwBNawnfd*M z3hzjidLaYLB4U{I;8}R%zZ5HL&x4EJ4j9%V!xD?)u;ce!{I%m6%XdBrHTvUleWD6q z89S0k9=MBqdkP7=V?jC^xA3BtnL-c4G&Fv0hr!t+ti~dVe$mcypu_u+x}#R1 zrHQ@ZV{;}pD1@M7qzDddKh4-GS76h{X>WWUM#|iQ?R6`Jod&-#eYq!^)f^@ohpS-e z4Q0Mzo-6dw3^?}c9w?qp!SnYo;bWbVIQF$8IlFThRk73Ilg4|{ke7KVy!{8IRE_DA z57Xh#775y>rOEI4Eo4dKt#H+gw5o2@;M!w_!h;%$+_4QYULu3__ITsPeZkCePdl1U z$;Y1(VpUx31vzpr9%_FnK*_N8*j*yehD|qywDjRPt+X9{#j+s&K|h;wOOD@slMNN? zqaY`=sYs$Xh;|TjK&cv z{*WD(0H-~DNJMQmc6~FY8+~TOdFCZvKk!DV?l+Cdeh3z(1U!fN)h7k9 zGRY*bB98sm=_cA9<-+*bS4>fNE!FFtMLZ9D!TRtL6G297@(fxogs@Mq95+}8R5v~)dCso(=^FP=lsK_2?ki#YvHIYjJOjK3TTD>AH=X}@Y14i@gm z*-uUA*Tr){{la7(x+noGPf7A_Jz21kokkCzcm}a689Tg8nKs9XLa=BpF}YpLX6UMO zjk>Au>SY$5J-r;;*0nP=DG}bH*n+e5%~9fCnN3{o6X+P%Eh8d+N}j znku~R6AY(Di%?a4Yn+{O5Oy0)#;4o*Sd`>mZ1)alOZBLry|DsjxvzqZ?GkjX_9=2C z(~|Gjy8tJm1uUYP!XC4$uuDD!-y066e(1UIY)N&`Z#Vn z=MGxDkl|hFvLG9|g8R<+0Ppq`fIxXR1}DdZn4cT(P1B@e(YpA%(1qL%yC;w@dj;~j zYBXK<4_XY{j8E^{Vc?A$xN?;(?Dkm7L-`wgQesAx_YVTTO~sL>C1k$eBer6{BSigb z5%gV@=bP{n7=3yo*ta_pjB<*htcOxv)kf@k)q}1fW`Y|#^4L@5blmuS3w-X{ghpbS z#II|yK)J0H1s^Hzm)?p3FyMx{gLva&7?~oa30ChqSn~e2WQUyv-PivJK2CiHy(QTw zZXJV3Z}UOhO^){OTth})kf!NPpULlgmC&MW46%);P`=_jEZjbfmuSn=^eYb`JtG1W zwmH!a!{ack5U^Nk4)jD_g1Nb=m=fg5TwPm1e!&r#nJov;42RxYITh9(oQlnXZ9hv@di9F~Dr!GH81v!8g{)5x+Iza3xKNNka~b+H9mF6dAH@$xp=M}0t6`_bP(4_72)_7DQr*NLzWSCn`rGh zB{W*O1byZkq30cC>QW@bpYQsCtpyq2+<$@_gtp?{AHyLcvJ*Szt>qVP&ES#s^~}hf z;_6z7>3O3hX!xY*kX`?mG>%G#=lOH!F+W0{-IL{`>-M0!j61BewWjV;dOSNcis|%T z!^Ou|a>*<;FiD<{Yjt0+`}g*Mibx-FQ!zAC8zGEo(`PT=ErxY-|Fe-Y89~GP>jjz; zEjH5|P54q*Y5v^BfNv9C#>L-PLaK=|yK+UH%)2N;byR*ahxHfmPhksuORU7bnU-wI z7cY`Ma12KL7s%3Ujxz0cqI7zC06*H0kJ*(eaM5)DmMF&Jo$@5(1}x!gMza-#~bg}T9{$C;=;m;l3EKM5m@QrP`{ zW|$cr0=M7qMw_AlO!fZ=?|)cf%MD2`rjbvkpH-vdgRUSyd`L=-yNt1+X0rX(VGHI^*+ z1M9LcR(e=^5s69bg;(5;Lr=Uazy50%?dorb)a*b!pOnX1OEciB;2ODV+szzrc~rSL z5b6{43qleCaMZK|7^L)y)ZDhBmj^};{l`q0HuVk|FTF)R*c5?5i74G4(F|G>646yY z3p`(UleK=Eg-v%u!TFg8yEU^MmWD;b_!K2-{%aUrpcRUPi7xoGT@AJFPeQekeRwh^ z8aqapvlo7&>F0l;7-Dq_SE(dpsX-8|A2ZrU?WsP$v^|b2v_DBq8V=&TQ-5I6`*q;E zObY5bJn`9}AaFaz>MJcYDYEF-9xo~ek zO{yKR2RGb(OrF$;fmr@~pi@$C?$#nW+VKgOUjGDZyPDZTxe9DLB!z}A6X8(i2HgMV z0_>h2gM$Ot$QHXAT==MgY2-HJr)PV?u)>{vOk(Vm(P7e=(GFq@=dsAZ3iip|6>nP~ zM6K+7Fm~}tT=n!Ent=+H{ZR>nbvKmoh}(`$s1E$GuYvyFOdEIHVlMjZcP0+ake3ehly5Q3=iVk<7`W z1`SuXpi#qH*c#xC`BN+5vG)mKa6~iq>FJqn59)a zR3`eva&>K9bHDOOmNKe+FuV^85OQx_t=C=jU zCZw_Y{XZb~)>T~Z^cn{i?7_dpTbaNejBtBahm%J?i(Z7IgqtBDxiF9U{GmFe@x zXQ#(nUB^Xw1g4aXq_CYDpT8;rH%-*$F^OB? zu=XbiJE}$>Z4ZYA6=O{PS|#u$6ZmiI_iWq}6JhmAZKBe70!~ixCu62DoVhj(cRDmN zvqh3n|09WrJedVX2@W>1(<}w*Uh6BR{chgmulQrB%6iY6DJL?zp|jyS@841 z0Iuz8We(zDptn++S8cwCLF5QOW3(K^I}}Kc^f3}}^ai_bXNX4|?cu*9HEjJ^iny}UUQ+PgYJikWJiv_(wJ=IslwE4yjMg`I;<}aDa85)( ze#KF^c~}>3Y>32tp>b^C;v;zcauiCQo63Jbeu8!FeZY58#$OA&NfIk66-{H@FzQARb3yEn%Y)b z_4YD6PI`|12h?%b>U(fDA`n)Fb=ruaHh*_a2eeexhdh~c&~d#KGD3c%@UjuLDgT4Z zO|!@^=ikEB>DNKZs)0!r?}IIq6Ja_T0C|Pm?83-7^yOj+nmMP3taLcVx*GbiR`IiN z-V$F-$w-FWCuzV^H;_lik}>pvDjA%$31hD5k(v>Ocu}t(-?pq|6N9Uu-EaeHB_9CW zW#YW^gdVsggyXTkK-BP%rR~Nc!qcX)EcecN!QU~r*lS}#nsg<3(WSerG4=rFJB%bQ z%>oE#qw!>F0vz8=cwf8*>L&bvppJ6*u5%i!|0v*>buEHX_gaPLpUCo$8Hw0>W&$6x zW(%G-5a%wLmSh)~5qK%TfShyR*ch#5wAgnH_T^kAxjpu5_;WR0SD_3mOcIF3l{&0l z8OS#Ve+9o{TU;z?L!R}5o!porm?ApTrp-1BRX@zc+l~*wS#2(_tmMD6oVS~-nfDdr2{}Gcddw`V< zR&aQ}28Q|HAg_Pb!VL=}@YaxoWZa8MtFzI6LKPA94Z+<;?o8`l1{^GH!&M8iaqOEU zELoh$RBL`g)fjg?m-q;7eGP;6Z4Ze~*B9tGCxsI4mC?gknQB@w5>s-Mb)BBa-Y5lN z#hyt-Bds4J?(PH2^fEYeyMsjBeG8g`tu`K8au`p1i}@+9Syq7_5&LU^x9@}t%&aL_ zTeO`;?BcK(C)3eSw0T0R1|Ntjfg903A=tDNBpfX0#eh3#*0KQqdyvY?3x{<2WHrn^ z`i2$Hk>=9BhVg8vK0Mf;2ZP^6a-R!NF!Qi3J@BKCX{q?Ko=Lf2EH#POTKvMz_d7|- z*WZv8ph33mAM()i9;5P^LcD*?kjzhw#=d_csF_&H_HMj^HS>SNj*))_L4PNs)x)FE zhf?TqR~nAG{GTHyfcDm@aNW?GR~=HvhH^O?5_b}>EepZaty}R!U4XF1rV@?Mi_`kl zulVs|DSX+zklDOfqdj(ZuALMAyV{qw-_xecSUo1JA9J*1jbbC zg|Tl(aSbj_?UTZV-It2cPbCzkM_W;;lLesvHyT7#yUD^Grb6Me7o2fEJQ)=k2US3_n}81kE$6jq0DP5sR%8` zVT+Q<*}1{Eagi3rc26LmT5hw~ep`vaGZ0QI9wx`w{hY>j*C=3XJ5__!sW0l82@=4WDIFX4mW>eP*^+j5!rwpqZVRt zixQ|3McA}UA4{ef(l_N5P@j7WW>}vFrOu0j&5|_`D_%#$4ll$rj;rbCb>Bha!*BRh zkpz9(o?L8FD5kcnquJ6fftQmzyvVqZRtoP(&e2KK#b6S%9eLD7kQ#(%NIx7Nx0x^8 z8bWI7-Qi_rhVZQTF+6f~s85x|;&FR1Vp*L`Qc~#9< z9$iyi;Z2S zLv5bxfpzIe67jD=nESFAi)F@Qt+FarQ5;VjL*=OYO%=!qpUS+}EFvSaN1*3=Io?xx z5O8ic}f%`fftj%{xjQwgut8GvDy}cmW2C zxrJjk^f37al`u{=N$~sWHS#2H0B)yEqxT+6XV<--l3fZKSnwzV*QVR^hy$g>_~%Ky z@h2F~rp`h6FAv#3=xnT;AP+V(Z)0P;4*OQ*!%UtHqRqz|&{b9kgDXmK^!IV_Ns#78 zT#He%tdV6+GlUSonHadt3QDy!;OnkX@Ys1|C@bD-_~lhINV0`FCQgcXWZcK zqX@X;o~_+U zFpp=D<7}~aIyv%m0(Mxr3THWqV((Da;nbKKL1@(g3wXb-YJ26$=|1~9P}md7zgiq& zt=p2x#?N1|@7|DSuU`N`_6FocVIWR-Cb;gpfQy`1g+5sb`4yhRi55|iY&V9L6{{|9peEU{xz9j)a zn|+CyY#+Y0cn&7vI^eMS7DzvjV@IkN6I~KZ(haU)p zK3a`phthCn{zAdQS0X%h`7IJuGYV_t_7KlkQMg>T(dOYCGgzp63zFZc@Fu5zIAi3& zugq+~M?opXXO{@Kh)zS_kRe{wEFnyetrcbsK7>aMFInDS01- z@fQ+M=InPk7hX%UoGj_jE$Xyue-hj;l%iGTDt!1FZ49xL$9?w-QDWXKn%R9sxJFx^ zCgN!l70JC>iv?*&VucX;vZL-aCbl z|2!8BrBaDY^-kjER|{9IRJhHlpQyTe6GRP^k@agk@!D$#$aEXeFAU|)BJJV@PdiqF zIrBl4rY&TQ{CUWHJQtYzkguJjhjEtLsP#{t8!oy`)?U1ak+0iGh2Cy_x4CRccld;o z-sZSEV;^xp`<-=bi|~SwS?pAFGiE)PfboU~a9%zc>TJT$rR)x*oj6WnCo0e>zn`E- zj3dvzbeuRZxC7D?jOCV^(U3j?02sFNB!DZLeu>M8Vkk>X06rV38hMGsA(_tDU zu}G%Zv4SJ36OR)plYSZ1S$eG6lUAOsV_~i?&32%(Pkpp9+-^Y zcQU}+sUAyrq{5n~jX2fF4?@bbA1ZO2}tcl}$aw%5S< z&;H`$)5lrW@A>ey_Cb}N)kt>vu|IaHWaFsnW>i;u36p;g91KVlZgu|& zBs_}z3hW|(wn;+vqzb%evKt-q8_4H7Cj4P%A?h3)OXdt)f&n4NAacPejJ7Dp2l=k} z`&=cYoD;(ykOPO}B0Ox|0^wb?crwZwen-D2E$6R5Nkkne%smdHWA$i$dZF;h!xb?4 z(;Rl(NLg?q^aR#LPJyE_k)W9DW;C|g_Qnh#=el1GI4Lz4&-9B}CbQ5Q* zKB(X$-!E8j(;SWO9i#kQDnpNA@H2@55e;v6+HXfV{RCqckDv+O^>BJg3rhE-B0L^W zrwqRfr;|tV)5iYbcQ77D#>B$&^n)zS`wP3G{~EsexH6Kg3^Ro9hpt1BdoHrX`R&ta z;&mmOUg9$71(Y-z$UavI5lO_A`ZjS;z zpu#aQycLo#2i*?^;;u7$ndGqu!5X(LT&GQ_)r&Slh1yi6UX-q|pMrI{`pfU4)2{b$gPq2Fn8l7R%LLcHNcf@s9{aAPG2bD-L}HsQw0IX{>`xn9aJ7is zzb?w19xL*SP=sfR51<7Pv&c+8#I-K?Yo-W^*V+Ta`+hNxiDy7#q7$t9F^X@?6VTQdfeC=5+9=f?i*r{Ajg#M54_$WzK)yO3EhT~zP{Bg3V%mZ1)O6+xr zfm4Ty1rrxf=i;9i@QYUYLbsa$vP(>C`lTk~!9(9M=-WjYcc=xDOGUYA{xNXcc8pyZ z^cEVAGKcdW&tYF#9{YB&l3ea@N2c@+-gbQvtlA`l0V<<$$>w#ab$JB2Z95LK?q!lY zu0EJ?yhK>&R05Bi#`CKsZ{YAF2P{yYj0?KQu=88fpl9|R?3T@9(uam~Tlwj@%Iy;? zK0Hq7*&GB96e=e_`(?b6#4o4#$qp#E#h+c(qG` zI=wgoA!qNSP4_8qzg7#Q|K#D5S{=SaCLOix@?m6iI;gE40O^_;v{yEpR2-2&=Oo=R(BU@D-l>QqmfAtv z?nvt9zZM0mN*LB$fuZ^{Apc_~+IIcN4t6|&PE`Z^tu+lLA5Mb{E`G?5p2l$|YVBg!`Q#N?EHfl2E*sc}qA+-3qfH|Y#XzxR94hY} zhHKVX(3vle;rUK+q0Yw1Xu86bl-9h3VSdAK^+rRe+3gIGV1tFyFPX*qR*<>fj7D23 z0N*SHC;#)z;Z-YhPYA@}#`}dY9HJr0Bn9Fn3#WfRp9%?9!5F6}N7gW z$>N1rLaz|5ZxJX%=0kkdV!S`AhJCJ@$@43;dEj(2Uaq@e(A+l?=5}*7c*GUIHK@?Q zzh3y?j4l!$UIz<0hx~1##BIG|@$<`}e*5b&vFO-C_PHq11Wkhb=M1o zR5U(xUreIkuZ5hsCz!f%A67@2k!hbkLW!*nc>JfsuLei3h-YK5t0b9y9=}dF=2jZM zah=7Fd*5gG%MOCELj^d$tA#J~ETLn`P0Y`%#vNHZAtZ7%xMSl zNexgJ^qOgiK4vz-;mlVd9*&x163w5%@L)j^#{AKyT_2?AzNBT`$2FE6x#@zRM{TZZ z4$#2d&c$G@bBEmj-iVpyVQkK{j~JAuh2v`OkliD+srS1aa3^`NeZMF_uvH17JWha7 zK`Xe3PNSwtd7yaZAYR#10qa*!!xj5pVToH6neg;FS+95nkNPWMfx=L|^n}B6w?;B~ z_Do`4+zrcX7*iY!M_B(4cTbIi!Yzb8C|xa36`4)9Cwd9Ak0+w!uE#ih!$|lR*UKzV z6|!r`55QcDY$(kA2!C?M)5#b9!v$nCy)i=@-+Y@*TYDa(NAnJ%*E$bW(hlR7G)=y6 zMloJ{8V&0H0njHs6+*>huyM$Ln=~gI)kLqLV1+yOSJ=SgRtt#!V9q+P?t*nX7QBG; zqTeS)Hl_C?&Wqa$-}>r6xj6;iJ6lX2u=NLL&l6abGZ(zaXc3d+Ml4EIhHq+;<@E_w zENop1S@e<##A}V=z?D>@5gLro6~b&Vlz+|1Dn4Z@Q1~ziPF325zJSr88ufJ;Mz7vc)DN_u~_N`uQV#L?|K_# zCh1c{uj5r|(rJRLxAmD^kF(&a!V)_B3CE9f-jk29vq9p>0M1PmV*8O=*gIYZ!S5DM zl=OmAojD{RBLVc%RB)>8IoKlW$z?qE(L>!4{N|%%5>k{7u|M?SQJX$~F&IMB-U6B1M zf;s7?aPU_#ykVs%@_i*;y1SW7=)KF<)D{67^+vEKVh@^?-y(UcQDkGHF<*FSm+++g z74(hW4knH|@ZcWBU2YSpnNuc<4oV~5i9NV>dlG)Aj(}@&XTa`(H`BLr#A3Hta_o#W zY`>(0X+v6lP22rOp3*tSq6OYxe|@43S}oF)bPfUmDse+0oQoV zpt%|AG0#X8JfegktG*keFLkm%W2-<_sDg1~=Ctdfy+CU7E!erO9!HDs$4A*_=q)jV zd&lKs#KUu75~511#G;@gP=#;*(LUsN`e3B$2@otQ!k^3HSTu36vAHls*!H|0&CZDM z!_PYKtGXl_9JZnla}7~p*hoJ0awOP~48!=)U+ntMM^N`%Ye+j+z-^l)sMkOQx;pw} zT<`=~V={%Fe6j`VUY5ghsam$;oh@q|n$w?!C7@(yHLM9TrQ(W?kZk)@_-5$e^p6UM z%!v*}_D~~emDPf5i3p!?ei0OF_=0Zc83>3!hSQ4k@aOgOtoYa=G_0}2lIcp&{AUr? z-|B(?8YW=Tjzp|bc#o69Lx%7-AJYae0y=k~U}`5MJ==yhhs)MH~&2usJ3TTi>9Ex2PVVf%PQahASo_^wfXh z;G$iF&qa)(>yjDuSz-YG9wLx%TO2-XhYPCywZXckyMn8q?Xc^{I<~j%KhnB)6cO9; z67B|M33HVvV2UOO#iId4wbq~Yx!GV`+I+m~J04f;KLIP(9<%vde-8F6{J}EE8sXQS z=TLr=1dpF+02d}j;NFh^KywJw)0f7RXSOChY2{9cnRJ?|x#Z)dNp>i^YZZvy6+ljz zJNxmg0E1h-(BJMbR9zj~BN6?MqZjnSyvqAzZ^t9Z6TgVtCrEMGKwTU^`3@+>UVtNNP38*ukaM__g zV4LWGD>Yhic}_Zv*l&mBrz_BMC~beD!JBJ5l;efz1t?izOV6ltsMoxV28}{o|0oaj zlvaS?>2}mxD8qN}^5rxC29sqWd-2BNQdpRs5BD|C!Ck}W*!!ZN%u1`mDOtN=O>+_s z7iPe?v1>s$BN|%O=dp$3Z{w=FlBhN=7viUepx@V{5cIPaRMp3$?Dl0a%X)#(@#R!% zwfP&)_IV6PW{G0hL`B;4b2R%o4_dOG(L%X#?F1%PiLy$ zfd0VQ{G4a6Ktpl~%{V^=pr;1<660{w(oImh-@t!$InltPSWsqhnN(i9#soK$Nh@c$V}=300`+OJAd z`S1b^yMBlS4?l}Kn-p=Pm5{90cmsD@caUY`pU5=(H{dbkOAT+^#0*Bekq1c%LaU7# z!Y%tb^gMTAEh;AjDax~nvV{tspFbNHm5k?JF?ARdV+;Ai9buZTC;$Gr8ePXA`7m6X zj#C;-N37~0fvJBnWxE`EUStR4l{b7#D8elw6*%sRC9Lnw1vAqo_Mv<@tC>FnGG`Qn z+AL&FGI6Z(p*v0#`a{0bZsr<&8z(FoK^>hF*tA*1!XU{>i3VQSiQ(2iL9Sj4rb{%TtMMV2|7bIU)Kez2R0@{< zZW+oad?dSbZsP7KF*th9FX+9MfZyZYsLZL0X#MUOejMVrZ;M1|*THH~dwv62N5+Hk z@Ub-6w~c(;ABM9pN3rV;$&h6-1Ag4tjGp=vX#Shi!q~e-d^`dn1WOBxKGhOVJ3wcAnes$MNGJm}f zbZd08hDiw!-|-OoXfgJ+=d!@_86fzO0%Jop>48VixK1G(GYogb42pF5tiXPwb75VaYrv+m<#T{o&B ziXJ-{~Xdv{!S6)Nt==|WL!Ni>G+JJ!fr#P?mwFq1}4HuPN`7B zyrJXGQ#hD52QC%eL6L|Dq`J|8e;>$!F0r$q)@#A$nA(%`zt=prD}~jqvh@8w z3E}%!mjp-nIjC4F3J+SZupKYY;>-$ld}?pZx~vLV^@Xo&{Shnb^=>NfTQY}uxLD%Z z^~NBbu^yJ??Es${&6qN8IwTwtg?5i(7Ibwro|_rRW$!iTQ_C%C{MgO8kXUdW zVlvzC{uV{l?vF(4@_%qGPMbwvP~%5G?LfYg2o6cirA@tckTY0_3K`?+?~$H(?rSBq zF0LUKTZS|~J6nOm4IQjivE|0?->_-xAtF`yl|}Zd;t97w=A_&T8@8|E4KvrEi`Fun zJ5LS&#vFyxhg{h8zyr3&4T6)k5{>9F=O>Z_$kLUX*b$!vwnfrB-bWJK%={p4NFP#Y zeZ_1vH=(7YJUToMVC7RBiKOB;FkD{6+->b}*5$WEGHw;Tx;P0w^!cEfQHpS`jWkr)WZfxUd7`r z_b@D(z-(nn7?nGojy_Wf`(zTp*KHm#yBvZKO#Xwlim#w$ObR<%@(q11lrc|_Vd(Ut zlPH`!hZnwk;D0Ig!rMdH%s{u%aJO$KZU2x4niH-wm(z7*#H3v?5A}J_ZXY7QAc{>K zc?ael>aX%B3MKoFE8~f4-dz2<1g@7p2Chp|$)nGfT+kkZZC_5}hh47(4u``)PVFjQ zzW)L=V=VZ=Qv%R>YXrf&{NcvVF_1q!lKpHR%WapIqCxB#T$OVQHf$83kIir5hJE#@ zR$WHEO`U`Rrv|XZ>l{q4Tgi=5RbcG=X!z?Jju}=4?DW#Zu=|1}Ki&Nuzm~ll+CjXE z1KX^*6qrESn<$u-wpKXu_YC2>Gty+u4O!S98o_dudYIMw7^o$i(W3H(AZbSpYOM+7 z>!Z^kDBu=Mx3qw!+POkY@oC_@CJvN>WwEBz41CuYV@v*~s`IuAV4zq%lo@`Ck1uYa zD~6AQo6agcxyFtT1emdJNB82SZ+``Si8Dy&-IXxv!!kT#Ys>mdhVm-gw5iK2RV-+Z zBONn>Va~qaq;yO!+-u7cCR;3oH-?I|Hb$IFxVj0PUcY8RE-@^lCK<<ZcrjI>DpW zZV;x~3VCk;YW7@UOS?6h>n?Ms+7|$aylu$na|V1waX4Oc?G`M0N{Hi7P5YKtVu0&# z`ahDcJgTPeeHTiS2Bk?fm(p~{v-dgomS{3%3K7W=mHHSmCDNo)X;LIb0|}|5?%DgC zdm}>LcdW+tP2D{ga zuU&s4KF=!2bnk^wob5(pZ?k++yEl~CEH#t4rwU~ewMDY=qq+0!oL8c}LaUlBucB)v zvS(!PEl)^?cFq&$RyK>@7m%FV4n0xr@l21i7q4}2<=A1MwlvXM!jrpms@hlx#% zwu)bG?vfRh|CW`jGqR(di^V}Fa>NH;ua>o$oR`fT)y8f7Y$WTQAr&WfNyQ;kBczsD z^F(_aF`G78i$4@%&Dzw@6LSt+6@{5dWe+>eYL>q|FCvDLq;BhCWY4*m>0$ z*^dNoS?8OhGO$I*(d^@68E?5#&F<=U=_S1oap|o`;t~CR>E4)$vN?eTVkR(M_Hn~~ zaqJ95*%HE8*32!EktHW&2BF2W;AZuj`86ssKgBz;6O#yOVO*c=rtuO=ymQ*kZ^t3=vs@qX z@h7H^jLtoAfJeD_zU6iC(a~BtLtc;Q((7%q9?fjo%m0RCy4jVZnn*0GPZdh5qbG>7 zt6N0jQ5H2jC+12Iuh}Xy@75K&?RIl4+0i2YbGTQ$HRq}Ls?JWCew&ljE_Og>J=)*k zU~8%PTyBxr^KzFoaK};Uf5$J#x*|sL!skTMOU+o(ELnp1!1yJyqJwGhH30N1u~g zPA=^gFicpX7O>nbh&GcY=mD!MdzxE`=#T>S${vujKL03((xnG zf-fs&Hx~fMS?N*Y!K|_3k6e=)ewMYkrBA% zW)Ztk>Y7{(cEEEk=9&QX*Xpds(5WK z5`58@Ue~Y?MdW5sNiCYvM|$?`w(Ll!2Zs(i(a$5DwBrsstu8+3lr=BPDQrQM)9Lw9 zPVYv|b=3cOIKt`0s2@6Nk}n6HIwB7`T?;?xM7l*ewT{~D(RawENGFZa=WF8+I$4h1 zZyEJ9OAb2ynj7WB8TD_Djy9uf<&Rq9frCyLqDSXhnu_jxHIeT6XC^gMG!?xXG7-gk znu$`NnMfhcOq$teCOTAPE*dJ{PsOTOh)SWU$alJ#RPLCus9aTz)VVxGZ*1gW}?nTX3~)5W+EFO6VaP1CL)_% zCz)0H)1}|Onu{!7Pmy-bF&7>BWiGn)^A@%2P64&wXLRmwV^Q2wQ&IaybCK3RU1`TX zbCK)&=~CB$Y0|O)3sFZ^5k7s#Tx2sw#O_Y)rdnT_i|!j+i0%n2L?+KCNL5Guu&3rC z(u85k0!9gi_2#0%k!Du@*f^=O-2Y$OCdZJK84UY;_Eomx_5)IR&OY{~e++fOH(vBF z@&fB>ZA0xIa%YNnJR)7}+n}^vg)HrcfAxNywf~XZY>?V(RabIe7f10#L@wCN-mu!3fLM;FZR1IR4iRrtnQOc}c9w z*qmR*46YqQQyz|CrYrk1)19nHOK)SwAe4nJU1O<#*Y=UCsSY9EQ%XiJ*-X~>pC*TH zK4+$F|3zx%zl078i%@K-ocMMBav2@IVpMCZOMM3iHhBK`3Bv}pQ( zyr}QF26c_2EAn1`Rw&~Pq>xZe7%^}d0)vk`EDWaKD+}DX^$tS?Ji{u-Q$>)OFx)zpN})bmg2rXEoMzqfT$#@Qo7Np zhqZQ35*<4;MwFIvL$q{Xlqk>bxm4@-2WnvBTc&c-H~9Km0ag08huUhOz!vUWMy+=| z&)jQNmZjZh*`q35Qumh=qz-=hBG;eEjE_Z6=mSqm?zTXbg z)CKoMR}RLoo_fjDhQ4U3>i`1-rvJu%&W>!u$~Ej&+id1YUjbz^<0=#Q_6JmCQkaSE zR#*dOk!OlNlHt4k*|kZV@Gp@j8{(19ZunS9R^$&b9bp#CgPS_c{Y-17HgABPX?vQ{ zd3zbzaje+T?~pN}CsE#cW2k@Uzmap42bi^T5157p)nu{lVkkVRP9D5GiM?_%ncQFU zo^^Q*8CNG0X6=)itWxu2rc7}xZr_pvl->2H5jL5P6+R+MHY-u$)jiC+z+dDcHiLN` zZ^!yu?;%r{YBGDKs7Nn4bukO%E<^2O*6dmP32f~06eetb5Czf?u&FM4aqYtrxV-f& zV>0g#1hJn`ri&xh98*CqG}_PjT6r)U6NJnKY6cXJEP?y}=7Ij#Q<$H)khwZdi!!~a z0Jo~>k+#=L$&0fmFdhAO8N72M>3hE!D;+*c)+*VK?(6Tc>tAxRy8D*+OBky9WT=q_4bvBfaD#iKh{Om+LBMHUtA)+lYO0`vyL%! zPtH-S8HcjHGJ`7qAt$Z4Q9u$a|53HcyQM!a{${tWJHg5|IddLS(Ch(fvG3uL9 z0yVpT9GlhS#5TmAV$O7N7)OH?q!ca>6Z_(sUY#VSdYLY@apP(7a9kwy=1eh}=`fwK zpr^BP>{;mdA+9?1=4&#x<2d#9@Bw%y>@=JnwVBPAWHakd0e1b?9EM-Nn7w^mpV~Xk ziZUx&A$aStNYHXFfS0-UK3_%sKHtZ$Nm#kcM$nmP#$Wd{jH@?g8}HqTJPyhl;oMu} zC}_rh0;h$Uyd{i~yTx*~P&eZbC*i|%q1&T5Jh7QP)Tvke6YMHZo5ljZ>s>?Mm_G`9dEZ!`vhh~V-P!`qlPgZVW#<_TW5Er*BhByg1^Z+U^8KR6+4iaFPPCUaEb zLVkk%Vcx`t;oP%=VqVw154_$JDf~67?{eS&cHlSU`*L?ovg2B^MS|OsE`CmJC1>G@ zWWl|?34$VvpZw+eg@Q|ut$CW8tocSToJTe0aO|wM@%z6V76=k*1l0kdJe4Pgf{-)! z1szldKljX69@YGeXCOa;d!|vFPir0){!01HiwJjj;MIQP=0CUOt)Dtyu*NuD@E4rs z|3o%|+6GqvxN(8|zBrkqQ#8c}bXxJ^-rnQBn0J{M>Uf^>&Q;zaCX~yWoVu8^+Pu^@ zc`a$LOeypF7Uc5om$}#Tg4ePTwpZ%}#e#LZh z%>R9LSajnncg*oCoJnsCxT&^r9Kq484j2CJbtri}hP!okfiU5%3n5)_OfaZ1D1@eu z1yAO#Bz7FB5q4)ECcb-~5IjBmf_Fbmz}@zrszA8-l;De`p10ZS3cvjBFP`zGF5*Y2 zqd>4el}IjZ6%3iH6YsaX^1c?f3y)Q62sZL1f|mCA0yTQIFuiL(zslN9_$qx1vE7#C zAMu?oDBXFVe=@3x<9non;~BD)KRlGoSJjl~ZT_w!i2pc;cX-ovPW!?xZc1i^1LyDn zx74$gcX0P{{vWR^4!&B7{A*Xf^Ov3!@Dg{Q<5e%y=Dztn#7SP}$^EvmSn%NP1l|i1 zOZ$T7hJ56>hC7(>iB}QTz`Liy@U)j-=Do@d=e=6Jf>*XxnX}gGC+Df*2VSGv4=%gn zl%O|!HdpJfI=8OVj33+nj6!fW4M%; z0q_3!f4uKoKH6K2jOUJPMnV)^!gbz-w>Aodw&^YoYb2`_)r z6ezSr3e$Xd@D^>F$cyaI7K};R#V`15z(0MA6+~9A;vG}05va7431}+)%e#xEaq72U$CF0oy9YSN&=7PTAXDMtT|88bNP!N z%JKSj=kn{PP2kP?I2vC~RQaug_WUe&8P`)&fYj?SDoe998yL^y!Gzj#&_}b2=6|t*I#Vtoi{~t8_){p^ z?PUllt9Qim&j!fusF1jlAP<$ex!B@53!V*z;XTw@;ssKLEoWZhdF`vwKQ}cZ!ny&R z>o^0y#f&9&{ll@aNg_~c=_fuk%fXf9g@kbOYE(Zf6GsS}p})%wdtjvN9u%K+NmZ4A44HstLUQ{l|SheX5#cQ|#!Tqt=V#HRhba8BW0+AHxRP?}`~ z8`}J_Tv)0wSDZ{+paQhne+$#ra2%zc+zo#;{zVz}_rMS@9ZPsN^!+bW>3bI>_}d$0 zXgBFJSl|+YA0Kii4fRve^$$&gTV9cLd}$0QT9Aq_C@VAT>(t=w^?6vgH5Ce1F2eO8 z*}^}fC)i@iOuW|H6?e5A!JD`L!O!G%aoqZR@Mwq!)I%F#!kPQ)qNOF zY8(JtHIE|m{6sYG1fCQMuB4X$Z-OU_Q1fH(Wa zqNTo}AgIZWP`n*RH$7O0P9!PO%bYDG$CTC)L-Z+u(~J{HWm!1kq+m;LnqDmImas&| zFc0`fzoIKtH_}r)#-i6lEAZ96+w>(`Kx?jCgueR$qWMJ>eWu5i=sX;S{vEwRHy(OM z=Zug<_%3hQxJg16ADRjew)~(wOCG<*qhSd zQ;lf%r;|b4lI23WIEUD;EkZlCID+`<6nb)21g*5n0BX|##7ARi7-_01_;0%$5KenX zDA!&AT?XMirD^HJfs8+3`(-Kc%L_%)2{9n>R5!i-)h4dXC8Ag+Kag_n@$$}yAKkwMsu~b-FQQG82;;Y73*^cpl$STsJ>wplM%tYnuL$rj??Q- zN1(FrTJVZRy0Aa%EExOvJaF0CPhVa#LcgE>kcgXhS8{7|I;fZ9!+QlU3Firq3B~WT zkazW8qBO4(?QBs+JAb~S)BnZ-HmD5zmo9)Ra{JNyU~80;_Z4h1;9_f^OF~aoN7&PC z1eAUVV9~31x^_~6hWMJy>Hh-a$ll6@JKAVE?DIzIeHP5w6Mf#ey|+$@hm zAD;sA|75_bt902Pjo3{o8-UE04B-0z=1R;;+Ee6IFQ;zOnlUb zs<+ppwo}T~=}Q9kQ-nJ+baMiAb@CPR z_06PtbQH5&&Y2|5LYcTN>sW5c3??R0gRR(8&(!$5CwJR+KrgIH4eM@U;tnmqi+zqU z;~h&#GQ*D)z3gBvXqZyQ1~H_IS|eHT(E<)!`-=>_4Uykb9W3{33szpT40`j9z>&8+ z@G(mbKCK#0Ywf!rX?tD)B5rO2-JCtb=BOK}{zoc~2)4vW1oEIf@FHlx)K088#zme3 zNfJ|a6&w_3BUCTd1Pij=pz~51+&owWy}P4P9^sZKedu*J88N{PfU_f+ zKC3{ZrpyAkP$?6HNb2YS2N8PQnT+lE&++|0H<;_ALEJS>mq@K!&}HR#I%~Ee5OYJJ z2Xzkj6paCH>c8oVn~(6qk2L6A$;FqZS0KseeMD@pGPbK4&+J?L5-lPXki!Fry^0;M zxxfy#J8Hm|_3OZMDUEu|)v#aM0vu7d1upfd27|O7zPiqeEc7fyt8TXA_X+=C$olu_ zR!tI2(8;IwTt9_su6_l%Q$C`=X?^Gi4ua z{zy#TntD&RY^ z+YH)$Yckq%aFe8U@=c^0Pz?p5UbrD(F1kO%9|kn0;G9Qi8E@~2a5=O^UekL~sfic% z{malVd1YwYmLqUi`wP5#j5++_@*hkJF=5`k$pUfZvl*SOhS*x=60*9aijTO=XAUY0 zv7B`kz06a{F~tN=VG_zTy~h@xG7!lbTT+2j#z{ju1K6f? zU1_Z6C3c|_hx#J7lg;oL?O_#s67TnxW1}8jVBbHBp;}Jx*lfo%N}bdc?T^%=x+~;G zoXi{4>59kf9?#26!q8h*<&_~-Xmb)iYCc8TX+*MJk0aUo$TsRkp_X)Y?RIt`uYp}2 z{|N3`dk1m?QqYO$RyuLQC9L;q1JkW{7LsxPcyq&XdKz1RykFm;H{o2#vkZ4=nQ{x0 zkIsPkyeg#KGm%-c?E$!A`5D{j%tvv13sB)@c}!2UXU2bV$4&C3AUf~?dRX}wd+DY^ zu>DH)t9BKj6qAPXoK3OjzrDCsEsaJN5yXEEiNJis93Edc0qJ}mKZ;GG2)_*&1)f|$ z4=fHA7^TVRC(n4oAAvg9;zZ#nj#q^TgC>C5WJOfgX^V?u^U!m(_24~sDOx?Ej0y2c zVQHuz8aLA#f7mt&b!^wfu8HR0Zt+7ToKr%4cf1T_e%G;tJWS|@Cll0M1<30^3$33e zL2gwvp81oezm>Ps1AVn%xnCCHbfXj<(apemXIRKGS0qjX_lOb6UH6B|JrT{V_ znwZiyjKT*4vFvp`{M1nfs;@tTqOl$D;2}k9JM9kIcYYj^u{@Xvo%aEJ`1~6xt&jj0 zqkejhZ7{yKDjZZy*J9>hg19>5A-0)d2!ALQz}Cg*am48W!nEBL6nci zk4t`N0b!mZJZ<_Ac0SS|w>^l%{_9lH)sPC|ZGkQfI)9xg(LaD@8(zj6{`SMjjryqb z4FJIprjdz3L+D!HE_i3bFTnlrt(pqW%1^DNtoanwV+bNLrM_hc?}=)M{`Sw_*K;&{^6>o|V) zHwWAK>M=JiSwm_0P7W!(1OLaL|}v`j>BPsnO=$M0Gg zGzBu(b#^1?Qa+xDwBd~9NGO{1|lqr(JUWT`rexgVO;e%C|9A6Va-vZ)awfB zsx89&W$D$i>(61|`R2qrkrkr76VU4eS8>^@SM&*=I(k6M7}po8W0LTM8k1EKS;vKK zggWwO>XDv@1qfYz57g)U6S_V=NIb5!Cz47MvC7w6`a^aT-Qs)<`HY`|w1*WWb@h74 zAb4!`SXnSi)0qu!YrX<2Pi4`!wa*i0rnI4L1w0hD@fLmgue~tVWjvm`dpRN#282E{ z=Hc`&kC0hr14;{$r#JMD#Zx|x1qz(SVBNOg%rliB%KLdR3D$bC58x1^-@b)?_V)p^ zP4PqCnKN_tlnT4%hCFq>DwS&VD88QYvan=C#27WbMPQ(^EXGjVSVb7%8(YKE3C``?de47m!j-p_YeDSdW? zDL@&GfBO-1Yd;`Vw#Q)En<`@9_XD&g(S}~ruZ1|J->Xe0%s@9fmkWQ4B+>T=y+D&m z7a(S8<3&4`(4gupc8Pu=RNa3VfmNl1o!}zUI3z}|AN(e&jKc817$fYd*iFYgrV)KN z4FS>>_`aS8{+s0pq0K_*n)Hu0pLQF?U7UfRg?>Q+YJK4K#u^l>vlLY@V-R1$uHRO2>GP?q)d_3MX;lIj?w{@jjsJQ|8AEqk>4R5p=d zb{B19(ok`UF2G7>(6LvR$j<5#q0(r9=C>q5PHilVSuh?=+|mn_*8Ae`X&#Jx!av}> z{5~DKEdlMlt%2>1$J2Mjny9{U5!ASK20xlx4^BV+4Ve2KINLo51S!44PRea)b(tD6 za%)AQ7uA_lZ~vj0Q}XHc{++nsMKNykISF!o2Z;?TdB~>PiLN|8AE~-{v9BK$P`>Z0 z$h&Gs+27L42=FJ+ElYuBVQ8;p(cNA(1zsKkMN%TcC6j>{Nv%~a}P z#c|5`<0-PtC4mYHnopfLvyRQ|HDeU}ce1*M=SknI&airtF=aG7j-_MAv36SmsaJna zvUHe5VBXgjoWoD0>!6Cv5~7j68*J7Yx8d+RQVeV{S@)*$s(Yi znuM6yiMai37~JEi2qIVR2j4#VBjs!3@WRvY;DYjGxV5_y*{Apb*HbmPZq+V)%B32< z3;T!WG`s=ZofDCJXR~C<@e`6My-C2UUk1{x<{^_LS3Jw25dU)eKxqAQ#8l`ou#*j= zfcIC4?gUFLNl-@%$(jKClJ^`ZvH>juzt)SqWah?Iuph z%$RB~5qun+ji04B;>^y8!1?8Jyl3@3#=B}gF3T9E|HLAEbms-UCuAW^Su(06*e0ZR z-B7~#=xm(PvyQo=)JFcSpFpyHPncLH!aL~$K*9XL;C>MlTYb%iWK7!7$9;wUn@R5zmxNo}xbX#$( zTKg+Pnb8n%E?1&M>!MJ=TW{n&YdV(xHbyJHt&oJB*+Ms7N+Cv@YT(Li6(Cl(6s+Lf7{{K*!>{IDF|$e59A5FD{Hme;(eW%hj5M9cxvv z$s1KH=`evZqb%^@%zT*Ua~^CfzkxnxG~ng6DqzmMTBP501}u9*!1u<}K((PYA)?E0 zX+H=5(ow{fAt~6eM;+@|o4|mj*6?{~I$Pu3N%?JlNB+*jZ28Y^?02~~c1-jNwnz0E zx#hMwOI`lQtjt)6Ppqh7M%)gwC9CICE=?bpf#Y1NOZ5ZgBE3Tj#$Kl~N)A(H2adA` zHl1Up>0M^O=k}4)d*bjsVmCGW<{Z{H$evx)kxF&1zR#L`h3uIZ*=%C-4m{H)7N%W4 zNnD=g1%GBl!O=DYIDT&dF5h(u{dr}Dt9L%3Ww$N}>NCgUhZ|CmTiATmm9GX}wHu&H zk_SCWz|Ktt*__uxR7o3-tecB3gGSsmFctfhG~)|T zAA-XvE1=DyV2lpGLu+il6IK@c1Qs1rq4)Y@;P>UJc&X7jAUd%T9r64nOny=-Fk(*% z>a05iEw^8y%t!j*zb{3AxAFpbr*(?n>9dM1D@j5(-W>oBoOO}ys|wn2`zpe1&O+p@ zwF>pOP6nTryhK+er$9>bdBnt|)6%qspw{m!emuz;eZIAh=n|!aNox;4kzphBTI!Cw z%8P{!Y^iXD`%UoVdNi^5#4>d2NCaMg@gn>aFbO$ZbP+Ft9>9^ZXDBOaEF3!80GO(G z^mUVQ@Rf}Uxp{B^B%eqox~%(wk-iyR(O`iS4wHE02?07bw*l-!lVDAA3?^EC!sqXb z!IxECSi!P@8N9E+Y~8R3F6GNJdweJ3lg_HlEXQJ6xAGnu($Z&wACDyy;+yJ~mpO_HFDk)3 zrnl(_F8{#KMg#C7I0r?aT!qH08V9ZAu3?pb`taxG9E@Yai4Id2_@T_4ari|eF6%&C z?b-{sn+?F_N{h*5O7&Rb3QKH@2?o5>`LJG25nS}EL&Dh4_&|jN>F$+<(wxePWM?tS z`w@UU+xm+_0B z@`)Ijd1@uBv0lVn-&%ry)Ey&3vsN&{Tjr2g|E(qKg>~rgTMn7|XEL#4c0KVPheE9! zEw~}!3C5zIu;5t+BN}4hsuL;rZM!?O)onRrWxSDWsSx2HS0i{mFqDpY;*Tr#YA{Vq z2`JtR$Xl+N%&RA7NH1w9^wtrR26tSUv?bbj{I6c{=+S<-=oN>IC{iK2m(68%Zr6wW z368jIqz*)fw-ULkIyk=f94_8^7mgh%9?fUf$Q$||q&jal9{0-@=0|KI)>mR+v0V!e z$ZsNJjGSTTnqg#E7>&DK4np1DWlZ&cXYBTEA`@`m6Z$M|h40i~qxu~#*kP1oFN>B3 z>f|_h@2MrUG5m!>%MpmYKLF*#l4HTYMlW5yh?z>nybP#+i2iXzZKOt z_k*`;=g>!~Ccb%s58E!UpcNZ0!xGo|AZ*JlFtqX-vG7_1ET0<*-mP(|8kbQDb&@y3 z+4G+fgVsBNbHZcLXulYNL@Tr`bRMPuy@2h`y2u=^Poj1|SEWuRou={w&8Zuj(Tvf{ zf25URIT>+N30hZ2ksdXWdRnE$dZ|lEVt5LB@^LOJ9_4pat|YQc_1xLNdK)S7=~8m? z%|Ob{=rjXQN`Yzz`_AjYH`*NZ4POw3JSD;NkIpE}ek_dpTZA&ken+kuBIq8O zOb>e(!j`=Aght3zpnmQxQF3Q7yt%UwRBdgg12?RLryuykUq$sq@17OF`Fkf=CjXwk zvqKs2&1Rqj?}B0Fv~g(Df_6d13`O*^#s{a}oeflWG$H=)F*sslF3K*HN?OA2)6bGK zK*qcXJZr-fVDL}^iXTqH=k1GyWi6-houoo^Bld=5`8f z26%Iz2I_vfh2F}RBHsBlg0nQ92s0VQsD&q>mi>Gh6rP5kpD4q`^JAEkPUd*zo&fDx zT8B)(PeTh2UJ+i}ppFCQ|3TwadO?r-T`=Gj36BU*ft_CyQ1YXPc(t0lP+7AIt$KY1 z*f^8}-yL?s^ugQ2#NBsrxzlRAX%7$2(+S0A%NF2w;npzMau)1)mB)6k`9n!m#!$oX zIqRYm!}@0qv!*+@vuoF0A(t!8W)}x*vCfUF@o#XK;hoQ7CzfxcHf>gBZ5mckRJxq# zV0I_zPqkB%?`2VjLFMeJKMeDQl(K3S!z4(kZ_H*)LB(y|@x+`tc3EU6_v-+%dwzYfSMy6+N0ecuBYfX=A0{ zJE$m35q#fj39->T_-a=sh`ySNHcjcrh4V$U=*9r8X}cJxTmA-V7V79s>O_L?)Q6cx z2XId7OMK(HJHD>^3|H?kg*Rp;z%8Z5c)0fz%KfcB`fpbmUM~6u+P23cqm!X%q%sD0 zXI_%{yuD4=B>7hdRvne(_a6hBRuBOGAtxd$vsh(lhAkLadsXFTuWN~HTZ4wTJz zLwmY5Ad$l=!lfnvSr~DHTBXM$vEg%Yh}uc{s6K1BZ6$fwuTv z^cuHx=n$?(PJa#X+Mk9J;h3+&Cow+(uQHwB()Bd?Iu-|QyaI`!WhgOLTN3+v1l|Uh z(1MWTaMqLt5SDaX=&Gy*cNpl9aA7Z~4gE)~8oLjkuiXiM2nqaR(PPxP!x#x0bHSw( z6Je)qDw!tDV?rJ3@Vcr9a$Kl9le{U7G;Pl#nGNR|Z_AG`!DTD%4_OO_+S8!zk0zKH zrH<_?KEZ~iyiv{Nc1B+JIkpOz$$(v3nJ_IUQsJ;Z?r;qvJFd0jzmy&m`TQx;ThIvW zM{^9P2U$#z8%>VUnGY9;-$ApZFYuAQp=k1~jR2gS4(Cs9g@*^vLbdU?ap)&!@Gkm1 zGH4fpw5DXjx26=KVh`Nr9|!*(C4s}!QrMRL2&Nx-j^wh|ReOitCMJ!kC2V~Lz|~pf zNQLFKV3h(NRq*%XEY1k9&-{u{Z|_G5v$OG{8bv6Y77s7%cm%iOEZ}(bKcs#=4p{In z;2iJONYaxJdrBrC^&4}c=*SoPu_hrIb2pDLpS%|QIjjc{hulZx&rtl~#0G2~r%71k zPeaEaOoOfK%F!Y#Lon`qBltRcnC-XrDA2h@7oPQ;jq=|Vz-y0Nkj+Q|6yLr>|5!Q| zms}Wx`Rfu1*?UgZIi;Am#cG zzm6jbz0#W~bn|E2Y(v3$yGnrLD_y)m>jY>D8`V}kaAgQT9`;r8qO3m5eP;k!dTE=f!disj-ob2%gNWnW4-EoH)m%PP#*C4C#^hz*Opq z<|am#-2;41r!&uo%E<8FMr87VQYL6nj&co-B)`jOQt&~C)V=ag65x@I&Ns$lpO{I^ z>K~>!0h*AN<|OJ%q2O`94Jd6~3|-))3%hUp2MhP_#<9yfQ3fl|9J{E%7;MpjneLuo zWp;wl^NlCUx%C3cf1Aq8ZsW24lX1i`QpM#s&rp2EytD+y6TZB)mVw zrl3ALIk}V8W$RFySsNYj;2XNenFOxxu>!|VFCmgGw!x?A6Tz)g6Uj`K<6!IBtF-g+ zm*Dt(HL$Ut50}2=(%Nr2BxiPHBSno$+>OOHVoSGrS;OoG8Td1D3K{=53H=SorHTY^*c(w_nR&0TQT_|W z)RVg}DYPMs8qQT`KZjaTXTt81=T)}A{Si`fRc;h@EnCd~H%E?=dAP7*<2UR?gBGUx zjf8F5naYlJJWfSR;>k5vtEfTMUyP>BBK)Dmk43DQs`MhM zVb_6EsO0A?l=E~A{Ol`-n%y#>o|O%F(f66SRUrf8axTL+_iusHcToc8Qgf&swGSTJ z-A4PTSONWrm*CZ;XTATCVlX-RxLwwPhpObyBbcDp_~gI>-rWNy;g;8)#~6$n|1_3p7 zbiCC@ikFplNmyikmvvNWG zrRmjnPC>%E!g6}Ojt(k6|4SGczmqt6sRevhF@Q^a82Wld1b99?gSOY`Cvv!M5_LBf zcs2JsF(=y(txB&$Y*Yw1Fm?tWsfvAQEwkAzO?wUcPX+rekl2F{-=79Xa@27++6T$V;1^#BZUx@%g_g&9o?|b zh$wv#gM^}+;O;no+VH0g?Dc3NcF8H?J!b3B0^8K;&3Bw>Vca*#(%6H7OdBI44Oc+p z6)nMRtzx1!C;;%IPt#lb$D#5I)9{~bABFSZ>Y%3!HV}Z)YJvmR3S@o2e+4Q4LieuUj3 z<=H4Bd$QCqkC4`9lb=QpIM}^>55se~MLNA!W%cE6z)Nio7Mx8~u7pgKZLFBYvg)q&?Tl&~6K2-ffdX*nv3Ug1^_0^j}v%>Gj- z^?EhDZLWippJcEc7Zonx`eK;uFoY9{yfrW=Vgd>-( z(&H2DaO3T7^uUf5@TaE}YWq!Nj#SSE6%*o+#$pcBI%OeTE*+11f@WhQ@ocbd^#Qz| zjbh$RTaW#n?NEX3YU~ykfv0YZf*<0mVeXs|FkfRXHr;v{OQvv0&jp(q^0NmPuQehq z?hli5eS*o#%&+JRvjWcU*MPN^fSl2KglITA1%i(U$O-eOFbO^h#3M}(>`5MG#(xY% zXPfpi4L))BV0S0X@ts5thMy!iL{;Ew!C2DotrfFM&XQzH;y}*4GMY@Of$L0E$vp;3 z$!I-ivMrPaZ!UjFr!H;8yPCxeY z14w5}2nkhVQnz^?4r%Vi)=yu+~d0;7faD*UQ44)=vBz}*G} zy1DWp&1Fm&IzsH)xXg_dM*IF-%rH~owo>K*?W|esth^W<8U{(k5EPGICSVU z?K>_N%orO6#&4)ZwUv4BQ{;49&5eeI~NhgL=1B;N5;J=AVH*4As1huQkoa zCM$!4vp>!IA4OLl2*vxxT~U&RM7cw;Gw(68YiD+mq>>~_MbW8r5UEs>gpxw!NQEvD zxsvSct|XyLR7yy6d?OvAOLsrNf9H?+=Y8Mj_#Dq8?Ka#li`H8y-4N0#X`fLeExJ}J zjaX>Ro^;sC+74@^_WFO}RHay&=G`X?RGA}I@$?ftba0TTJ$S=~Y^UY7GL_}YC2K`- zc@?twKkd@r6)oID0Ayk3WwMCZg`%OML$ZCT-$V^Ys^VoSi$td!jKpK#YKl!i>&i_& zEtO8_`zQ0SB}J&=EZgLBM|4F$PIe95mW@%c5G&<)%Ztt)k!7flk`K(*7S%2muzJnC zoKnvVF2{1TZ0DC>(($*AMBlT3^j!qao|MIN{YmE}S6_B8lj1kC$M%Oy^*sW)lvS^! zI)`jz>KBKY?T=PU$6JOA=Sv?9V`OKzLuytswo+577pcg0zuU@g2?&vHEvH4LpVOHU z>-yNo)k9puy8`Jp=hxCzbz7x>3hJ0?=c2h+M)z3a#6h+%;EvSxLpvj=9w9xUdx@E; zIahcy#7a`%q$UlIZxG_YCd^EF9vjiq%3k?9ATbY3VhxU5=H$z|ncO>V!cc`XEZ@$a z5%DH-yQd{EuJ(|S@6FHaglTyLhL6&@+!|5LvaS8znqVsJg(zh#D zvClnhSj~(plID+){hl#jv@vZC+u(Ldcui8kI*p8HgW8)o&mAGsy4HhC#qyfXV^fOty?K9cmUsyJ|car>*!aG^xR8_GDFN)n}+$t)qSCJ9>N$yaxb%AnE6akC@e?!k9Z2f7o1SZIQ`yby;zj zGF#*QmJ1O-W#EIWLVjinXQvV+4QV{dM*T|^*51u#SEx@BMgIN8MU5HX{%xa8{Pul3f~Bm8qL#jAx($7kVJO(lfb_ z)9%fXR@bXDQ8mk?rnw$Wkzpz4pIQm|A^NMrzm9n?HDRx4!mqbVRsD#aplve+=m3cG@*`CL5BrW-g9MLgWy5VE0 zWME2_?WE{^EVsT+x=y*0(bX+s+!wu;)_fjdmI|_@>WQPJj|g|$8JZ)cZ|w=0L)uiS zU9P!A>tLDmuS1ITk!2|}&SWC1xj&V23W(#vrf}ST^O5qAU2Da&=j{=Vndv4UyM3dK z$zCIGIj1Toqx?nl+;7P&+FWG02QG6fPuk1s{ol)EZyH6eOODE%?)?@?(i!nSpOvC# zKF;E&A}w*|O)L55Rh+c-ke+dnFed^ckhs z0@envl#ZShB>eD$FO4CFq$9q|naeMRcZ}S0msl7a;A#gVx#g=BWc@)(Qn{gwS%0eA z_V2bBDKNdo-V@AW&Zc;<;Kh9Isj`)HkGq{z9py?_#0D~t&PTD|y$-V$%s-~*6D<`_ zFl2-NQ)Z*AQW?v%$wIAP-j#lRYTUS|_UyX$)x#QFE9Q{dBsMC1DHrP_W858`nLrO2 z)3G3*2~#CR%dIalwW*ZQfY`>aZR=ugh4Z*&TVokTr#|NCxpTr>53VvuWrOr&p0;f0 z+AhYHlt^2FCmWesA~M)#$cZ``W=Vzy`$@Ez)eT7$dgo7ILo|UjX?-^9?qM&rOFAc= zJV`?oT$m-vsM#P|wC5Wmw;0KuU&1kO4JL~oZ_{Bb_7@9Ztnd?&FUN_ZYCa2llkHh5 zb~Vdi*UXg_`*Dp5qwLc5Ev&MrT`u3(%C|cm$B9eNx!D;Tb&A2HdGZ>Ib7G4mPqEUX zJ8WMYDc;&KAl_svmoHtuSsb#UTHazfw@TU8Sza~~RV71}s{PwE?ZW1%iqBm(vzt@A zL~j4zcxJ=x=W=OKi8yIWnfSQ=xT*=4X4`2@x+o5|)3ST~FI_6Rpd!oXB`{Cn6z=r= zqwM(kF(Q>{q4Z?c3|67EmeE@9i?PalBn*rUW}wV|SZDW%UEwuOx<0Fr%eGJzty`YS z8K+u{IK{)v!G~4MuI(o51wVo4d8ilrJVB9*Xc+duM*d^P7d~^YQ<9i^p&lD_*@FrD zR>lzIG}bLBkuB@6=YmV@rJsCPNrT4MbLDCJY{;|;(oMh4a=tp4TWHl#xne0%DeF93 z=`?Q*_h6`8IDd@|XZ_|8XR2etSe?2gvAp%XQh2bIGji&a42*s()eqUu-MgPCtP0hV zVvQV5A~t7|MqlGJZ8NxHyHeXf!!_etg;X|>>SjHcuH#I;#Im=#W2D;-9hIeqJZ7FQ zY?BlVIquNN>C)Q1lM?fwa?UVpE!(i^(l9ox%^ttJn9Y5pB~x7Qz=ZbL%3hg$=hzDs z+^u6n+!#|r_Hp|dY38<0$#xrSSx5XA=~=DxVQey;`?ldTXJ<<=pN5Jh)psl9O&+7{ zzP}wOPRi_&U!dm5bvKQuD!!O0uZXc0k4ak~zv0m*`?Keu^!C;lviDcg<<@e4@uR0k z@c9l3xp*So4G$sqN=%rgOM9($H6u{pcxUI>&ZPrz_3jv<=d@jxTD` zSCh?UUqgA)m7QY7Xz66B_m~E0_~khE;GYABu7^&Z&@u_p>vA)PqLhw zo;!)9cU|MwN!+CiZ;fPqnPZhX)wx3MLnU|TyP@P+U9iAW(V9I|d|1l6b%2?myGY{7 zFO~+YB}-mCe=q&YpTT(-`U`6`I=Q)x-O@c8HzjAj?B#-jw@QtLmQwS-6WNF3{s_Yt z=(1%^*4%HSFm7>woK!E_j(Jg}#&o=2z})zo%pP!i%-9BcNvC^Sa(C}%a{LSHnD(L7 z!n^Im+~d0t>Gh#~?97rvp`ob>8zQ>Kep8I%Rt>2!#>z@ee$FgzX8Rb{*r1wS@?a{X z!y}k=Ls8u5y*^B8Xohrbjfr$?#eAv2>Mvt3dG;`mqAPbxIU){TbyyUzG*%va%0*tH zlOq2rYLt<#%CaT!i_FmXtL*t9MQP=VjWV+-cJee+EPhNLlYKv^Bo3SwFJ5Z1RupA* zP~6=>iuGQ5$$$A|$U@*!xsh40$ouztSyi{U_`;!ovV&cJWPaaB@tU3(d5cg}Uc5M1 zUXk@cY9ljbvo4*nWlzPh<2T)54Yzt$n&_2Er_(81)QsaCwn!6Nr!qn|{uB4PLnsY5 z3uZg=Z!ilJ8rf0WZtTX}U$|H6s-?f9?+L@E>>uX)RM|J0Wo({5W@O)h&_%ULTGp_Q zGoA2M`g(1E^t?fV^wUUd)>9>!{jR-*)zN#;JlfCaPCOK``8;)L!qcVF^_v&4s^UZ3 zhS*-_Wy567G-pO-`K?)!mccYexh$6PviUA)^*P0{>cP@?TPiD`y_zRA+&}EYuwb}# zX>#FQ%PEXab)?`);-K*KV@pZ$qCPJ5x(jRY@g%o-ktZA5d|4>`=))BnMzO2WFD}z> zztF^;l3smpAk`cDQKI9jBULaPBN-J| z!S!#jkwh$yNn7xW%X;^To5(j5uR6a%u12qw?arSjj*pulex^2GeBxKV$mF4gEXO=YBq~V| znGAhmhAm1W*D@XPx7Uwlt;WkmeEWQvP_$kC*TzxyAfJ+JdTYviP1VK5A@fBIBj<}N z&7)+yaR%!kSCQ9owIZ`U*F|kfs&d=+ZsHHqnnlzxp7`nWFqW*f-~#xDT*)V)Q2Ik% z+B{N8Qc<16jz~Ji)xCblyee(9P42Cy?JqnUrzCY2gxcgG}b5Xf^Ao(n=TS6|(V7In0KDqwM0t3nevb zuei6fl9hw@($EPilCa}Z(!iq`m8%aWNZO}Q=X?ftN-K6HNe0(mm1HOA zN_FNw;f!r%%$l>$C6Qs-OmBWvvcggsd$|*sa8pvy()-XSlz*$Xu=%T@QSP1MKLEf#50vMTV-eL zr;7*sTts_YksPw^G6m!;cYj|XtBO9(RLsejEfXG=?QfQH4HG8I-l~82zlTyZdEH`} z=K_i7v$IfK{lrRip+F#xZ|@f!@zIit+{|QMF=OQOAy1UmBbS;MDv5Qd0@>Q$0@)VT zzoHQ$Rt`55$(E<<$+PVx+$Nq9%XKZ4Jo@l~$;)hGzRWU{_P3nl{&XdA9nGp-?W!pf z+dtWYlg3`0{}MBf{Aa{?r+YGg-6ycW=M_oR5>>e7u5Hq(t54ZpbG7E8XKOL7OH71FQ46@MCk>tu;lq zpMw*)-n@ABxO}2?O4m~1->k#Z=~c6(?|cxGP(Fhl8#_ywuX~M z^5?Kc&IhHsyjIC3G(vi*Fi85sdZ!tJ;mi;OJeGb{^O>4-IOZ*cb!|(8X%FN zk*sXhOQCk9D!Xx19IFy?iiy5!A-%7_leYXZ5MTVhQm(adwM^qY7B}4=D^|O{R6IJT zPW1dfCEF2`A?p2jKr|}l8FOd*6j9h14RLL4lgvWAT%=l`DVq=)El<1YAY1)ZEUz|E zkgGUoiM_|Uh{BZw;(`N_va3gCvBz)zl%*fLCwdJph#v5i<=;%5#h-^7M4cz}#rAKD zScTB zk`C@(^)2bc%_-7b$)}kSXY|<5O%Xy-$py*wJ{L|`Ayq0q`A0Id--R)Lbe0Vu9hpSm zA8a$D$)41{$6kT|xTv2IQiHk0OqJlP;9Q@Mq%5;k*cUj53s|8jnLK3{qj)y1@`=+N zX0K1a?Lg=`>a0$$;M|6eN~56jOwH*LT#$}Eci-o>1Z{{CoWIZ}T!T%R&@h?AL%(0> zvh*o)D!bcuN6#H$$?I6=Uf*TguAh6DNA{yQuR>qOb5*kK@NOK*nRIpb%*!?0bG?cy zW*=l+gGpcYu})uDCO9e_9_=r&-O^OKm1ddeNsY{&X^PU-4kF3Rafi9F z8`Py9!kJ7);aq0BvjTT)_9vn5`;DB>%Nf#*UyZq4;bszgn1)X{kzG09aFq#EdB&`m z7Q_^}ALi=s7jYV4NuvEvU&&Hmy_DMRs}ohVpB82B?htLQw-vq7sN$rowP>G%r6?;- z#Mmw=WbM^5L^ZWRvU@G>SQi(8%;xuZS>EZ((r*Rzvbd#jvadnMMGL;YU~_zjafR!z zq@xDcu?L@~$(CH5D~f$&KYRzGW#pH8A|+0ey9g{_X#Pi^{A)xgP& zk5f96?0ehROrgG#yVfpgjM0$xo{Ex`WoHVf9@k+vE-)AJXGgNj)DJPehn_s&XhQQ{jPk_=zb$s*BAv7b@6uT69!J?1FFnLNPI8!;AsCN$`R*l>b zH?+PZn=@AeiSIO2?3V8BG)(C(~Oef{1m_#OIgS z;lQbJ_)z#9*gh;nTeL?^zPa@S;;%<=d2AOz20wTw)C1t`>H7F|^8qCwD46&yHmAI{uz1mxtcA@458!23RT!q+aJhr6$G_*T&csINCd z16v{Wpk^eBow=Eek!?paUWKCQQ_hIJSb~N zm!40i#?`0cu%oN6cJ?F`b)y{l4D_In`)tYgpG;ua?l_!eP=J`liRk&2XdL`;DHSh! z20araaPQO6SRf;D!uT$v<1vI2#{Ph6ojN#5!v!0EI*9@bkE1OEW2ma%d01hd98sP= zIHafsYd9F8Ya8@oef2`HXjt7m&3!knGpxmjdJcCG*Vqk}>4IbL>0!!#x^i+Ey*}X&^Hi?UP)0CgU#s9qq`|8?2V8A_6E%Y+6bM#su zZ+*tI8+5mEnpQ$QG`>l!d^85OCG-}m>s6zb!957N>_odGv0&j z1!uvi@ge-!(lLS~>(uBki<|hR^HupeQGNWtbQS);pfYLMs3+#$e+JmnNIt-3F_BR z;m2qc(F>o~)89`7@{6o``Hdr6=)rd>{Mwm$z{Tl2YQ3FHNUphq>cq+LTc#0?pDH7h zR?Q)QdhP_Z(+`8cs)f9ry+x#`tN^U;7a;HI4Rr z9F+pFb7CfP3^@tvJiijW%1rbz)))?}f1b{KRzLvUyMZyWynINq_3Zg6DJ15kcDfk@SjCciHU zLbG2?CvyESA*yIR@pkAm?{jWES#o|IbjUF0J+og-TJAUH>^^fM(_tc`%rH|LX{$vfbi@n3=N^%>}lp8`pg2NNTmvx%l#Z%9MW!!~NG zbjb3GC17K~EKs+;i%_?{PBeZRiL}&@5?NiYDEglx&w4~TdE-VJDLuo3{6&T~6%U7q zJhliH+d9JCduu?JmlT9-9|ar>GYRK>19I`y$5fT-530Av6f-!Rs-&#(C&jze36s6l zqShl+V}uv}qP-ejm|X#k?><6op#kNwg~W~62){Aifupqy>5;2V@Xxo4DO9wJIyJ(W z%6b?I+wXm$Uf)c_PsjJ7v4`XE*Y_9k!~_m+x7<$^>{X?m16Sc6FDk??i4ucyBOXn{zm5ZdJ_li>tJcU5viP1KoX<6 zp!2qMwa@Q!&>V&7@(`jpJ}y;|MAo= z%4XeY{-!r8=!pgk_*Y{0&}*6uhOHA9selbyl<5KzPBd!A1Ftjb9Gd~k>GTl_uKhsu zpFPd5Nye1^Y7PDXh^Kp~3cC4sBi^x8jejI8hx*c;ihf?1Kp&cg>6(xN3Vc0B&y_%a zHEnhzm?#Q&H>({>0PkZIg*$7TM_Wo zcA~mvZm`PK5L-QvgCrXjY*al1evk@~@6P?mH2))bGjy5U;&}@=?Q%nnYxWZ6`TyXJ zOHE|X`VXLWbnLJhBhm)D|0FtgjUc9M3xQMKmV>mE`|wSz5&ZB9k)c6`&^YxGI34pD zq;1M0>!})2<^6afuoZv}mCK>S&bK_~hC1@q?FRY5VQ>YdN-EgQhQbL(nBO=EUfdWS ztjr4r*>B#W5`%qM+#ihGZbuP*-^UZQO)MNZ`ha+BW(j29FQH9`jIsaDdNQfzGr8+| z6u!S}HmtA)XmcfiGk=dK&rJOfTBJR~Pm^Nc%CzgG*;f$?-bbKen>U&*Z?N6*7Pw(_9|qYa<9DBXP-A!;CJoP! zQuBRqY^wrv+54O5Z{|Ux`Yq_ZNdv0xiU+ln1s-dihF!OW!-K|QKx3N^K~3WW-Qy{+ zwW|PM0v^ybr;EJ3^bM3bUWQYQB-pd;Ak?lKi=Rxn3A^@nqoiad{2|8{o2)1R1HA}2 z-G7I?j?F>`)k}~_?2gbhHK;s1-kF~0NN81c@V2xLk^e?I!iD#GNgcr*zz7`B17bfY zyU4)u!^LEaPY~>UGzS>Gyovch6Kd~GAuatZ(eKN07(LeoDqnDhCOxTOT4Xp;UOx|- zhPmO9)p>Y&_foK%yND{T^q}=4_31Y=N1&o43lOHA4E^73M;5lfxkSg} zf=K|C=#`@x)jE`Q#yHr2!jt+|KMEyvT0$Q87Dkt>r!GF{qr;YK$!%j9s&eWb+!Ou+ z_!|a*uHCvQFMKQMrSpay%MTI+434pla53ON-F{1;AAEvOj8n1|{<9F=bcKAj{0x8A zqDI3pi?Gyl-9Pl7hnFZ5TLTrhH}mmuKDF2N>yNVnZi z707?6^KZT_!T()X6?7b};M;HI`0f0If|t@EL85K}f9c^sLC4xSTqE9#TP>f0-erBH zLETim%_EUY+-iv*TO2_vJqw2){Wv&lQ!#mOA|@{r!3bYjjF<5RFw7jnJ+1B79>>5- z^hvy2IRKP5>%cvU4bW(X4b}8$3ED%ig?&L;(9dlf&VD$A6bxsBvxokI-<$pe|Gk`m zyeCW1Yo$@-rhngw&}V9JK*t_IwuS6I*aOFq5@Klhys|yxh+rGJ^_Q9j^n zV3WTwOep#d`!??d(dX+)(HBcJaiJzWZ(jz|^@2gkmHV(uXB#i-{86ye%pCcDY9k98 zR8Zp|5%{bc4twn5$&~@~VU6)}T%kUjEM2Wc{)e04Dc{NXqC^Qtp16&cU7b(fITTBF zTwMwIFMkm`parbi{1?rPU4Vn{co7rHICyn;3hwgCgDdK#&|=V0W@FL09nRumaHi#DrO;p>W3DASFPY44S_ zw0guO{^L6bsS)YH@NGjr)jZfqIm&+k-9x1~yy*a4zgn0gA=GU;Kaw*a`3>; zU$mVnkAAT+nZDHmsWXMwX_vQSDd%kY@Bq*R+IE~49b1}BO+T(l_eOr8@1OlZ`8$@; zNwS-Gqum^0(*jQneQ0@WT@fQt1B z)PG)%$b0TqJif*Vy{a_B*QZU!!f^u9b%`(V_?m*I2D;#j3wI;WiUxG&t`>2u+#JQRMsC~N4&C2{ z<8%HW@N|c6G-;wHc>O$%aJr~~BK+o&3#}VL_g+5!P<9rt=hVTKo{gXMLhUs8gSW43!I!pwK}FY;;3ebH_|LQ_xFz}o zE^A7}OD653M*k!z+pNu4cATR0;z!WiHkeYPwo@>4&10%AZX(`wMSvf)>rg5Wt?`$? zWz^0HbJ{a(3T;VtVehF~lsT}YW+g9#s~081sm1egmPo~Y1|DVAi7huBfX_5FsWn>O)G2pIG=9209GNl^>>kZRy^zbu{CXLs zB-BF3_a_qbO;#cw=ZPrlRV>xFb1ka7@f3?Qt$a_@+GINRmheH!VW3V2ym;T`m?>pjsyZi`l@f?8$YJL-03ou+~F#+f572A|w zHo#jgONqT#jln6<43^)g@cDpvvaxJ2nbbT6Db~M*a}V!ARmIQAe`98#nQbq~Q2mwg z@JpZN01NwL&;5t-+|63eQ*->*hF~zBgGwic>0@; zfsH#>0sVwjxLiMmoVJ%B{qM!VEf|2hfpBunLY_^_-+K7|@pjM`VnEor`Jq<55O^kJ zHQcsMfwVYM1vQ#)kU4*2!JU7_w4nVMpIVVgpMT;FZ`83>NVKnsbk(>D z6|R_phu$*uZQ@#Tif<6yqgqX5Rx5z%Zk0UYJ$*3#YZCA3!>8o=h{IrH=n1$x;4}Ol z6iQ6rsDu3#70JV{H$duyS>)y&5GM4v0H1_6!NTY%VA1zw-_mOjaq2~nJMCT`ps?_95)?cUF4_v3$tUbmzZc-Mg zH0RML>3w`_B_r&WUxW7E^94IjA+rC+U37fb5XM~`dcWxkT)a1h$hrC$ggh}OWzDCF zhtnpamG1A5lgAq1Ih^mFu3Cb1zYtKlx*g36X~-r&K5$18`&!u8SOPs) zOe0?m7^BDko|226PVK+SA_rswdSS4W`LT5XP zt3m6b>V!xf`y+-bm}`P#T?OQi38z70(gLKl=OX!fQWB`J+<}*F{)z!O2#xJ9eyfC2VwH2V;;{EV@&Lhxg(T?HH(Ny4y36zG8 zH6nt4!NmnjAfa497JZuuZ|r`Gi2hNy9NZ%xXsyH}dMN75;~(&8(>0V=IL>Bbk~zHh zYd8Eh3sFg*w9qA=Lb%N69L(7>h|GH+<@@p|bkK;#C;WE9EK_|vOV9>$4k)6fl`2HS zI1!9UlL7N~03B9KKr^!!RQRf+`w3&f{4WP>(xu77`uBOHlXeg~<2@aM1IMAq*{9?S zlXhbN!b32;Y9r)-e@rf2@C*D_^T*Xh54rY19&pMmLkR^}z|H=P(B`fQ(7nb(Nmd2K zucTFg<_Xa+^|xqKmoIr<#|n2R-GHe+zIbq<1!@YmBS$Wm0HsA+$)EZmyo`)i_(Jps z>XVizw{jHxa48(Q)`mgnQJyfPbS1u_OoQ%-IZ(gS2!6Y?64l!ogE0h2wEmic|9J3F zn86)*+~qks8u1bqOHGl$Nq{oj2l+479}(nEOs6AP%oK!{7O;{-GMqxdW3bLi)P zC-4;)RnVn+iRkxFL|@Li!_Vjs;Lq5UOMiOK@b&6W3Kl#1(cBq#LEhP0e6tyU_|CIU zXua`M1=^Q0X#EjS(Bi^;e)TCYzGjsT{|xTsAC_+rI2q01cfL{;yo$|3)7;)5v)E+v zvFnN9nzsXabWX>`xjNYMiVE!6WP;*{Qb?60<<>>N<<3c8R=~_5mW_JJze)YnV{1Eu%(Q(pv z(F3U4{|2`tHSRN5w`~+P&o2m` zn0yaknly$MZT*Pf4jJO-!biATVK(|u^9)bcDkDEx?|}=BsZzf;RAbcafG&vtLl2%E zp%gS==#etL5%esM4WulBN~B!B8*X~#AE-=!&((-ITxviTD7zM_mP*B`(I z9XWUvV@_2muf&s2sexMwaUk(*5Y!i2<4N)}sHjh~U+VI2r zdX%=^0Z!eOj&pD9hvrAdVVijlkhD6AWE~%I(~6TwPCCG84yr&e?7&)reR)a9Qq||fuRe$ z$%wvEI9j3tJmV(7v^B4Q#jfjcfz=YcV7nTsU#o>TWPBvFU;3kuPc5jX4riFNbt>WU zNs45y1$frvZIpKRW~@2rL<(PJfcBoXs9Wtn^40RS&}vE%{#(SSRJ2CJl>V8danO7! zWJfmoG36ktEJ;FTekp*;s7LOl{gkWP4K!k{7HIe~2Fv`%;#u;OXxxO2I7yUBt}2K| zwaLwh-#H!+EZIkOw(#)wbAyy$S`jt6VLzp1yq|IeGAOCwQ5&icAw!c)u*jolNjuh%S*?{WWPT{ti1?cAFCUkMvH@xB1 zD`ZzHq|E(4P}}Ef&|L}&*x16G%DlFoGK~5M7ymj5jsDf4R`qGr-k}a`8@QQro2`h1 z%a@_=ii;sP;tM%_vJ$#=-UJ73+JLt|wSbP|?|AXSja0*=#prUS1!7O0Bu>0?h0ppl zk@DGesyE6NwWYN}YxO?#?{q1W4d+!K7eADr(SvP1MhUNgz|9M)d>rPPQDLm)!t6udZ@2)nARjWdOOAR#C0P5dfajr)@VJQM zXp?aixqB!FIM!{0zo#Du#TMaUlXDC|Wr?-mObS5{p8Uz*VXMr~)te~zb8afXd)F;W zB&(wr=?Ul%-UnFhQcG!f`0~^6NWN#oX8Iee!|(K(AW&VUOb?pB=Z6h!;}@OE=BJKH zq2%u_@UyRI(UDh6;W<4QemL%+=f8hKKdMgS=dRKgl)vkt%P*hjDzlmtd^9aJ%6$*fP!% zXNfh@yN~L?>DvwRPM{TB;k=VfQAi>M3+iEcs0sdO5kpvyx575TgYfUdo4Dnz0rEK^ z0@b7+Y&RPP+-}X|#YXhOvJYkOpHePdx%&t_bgLcJw+Y~BVG}4$+=cy9-VootA=C-b zhaFbVaJAO~-k2pZe27ug?A3%`kITNWZw(0&3`AU_1Pc!*Jgs3 zp?AT&yMCbR!FTZUU>u?PBo8>uNrD~qZa{g3IYvWc(pLY^m@A zn#U^x51dc9&i;V{_x0dITfC6^z#QUwd;lTYUjVE8J`+2=3dnM6 zRMm!aM+(lhQOD!xY^=5bP)n=Isc#FS@YWz7>axpmDk^n3l}*~BrPZgX&TEPIz*8e? zTD237J$4L>M?A$*ieD(|EJL4}nT1-OS7L%YjAsv>1;pQU5I)5UWsOb6b*6e)V`>0y zF!Kg?5f3KscLD(kO5`|7g zt?NLNf)GzqECAl;4e`$4N95x64d~px8Zz)2;9V`4Og`v~2i-OQU`T8@R25gE(Q^xs z>N6#xr6vZLipQek4`;#=qsD=*tX)uZk1o$Nv;p`Ssi5}yxZKU_57faq^Ju)T)C7l&(Zg;B7TCP}`-BL7eiYsq??DzN6v5+K zV*F`U4L-6$iFCZ_3BAV`=sK9u zbpozUpN>BDEx-e-Qoy91VB~x9G_pv$M2D}^EVgWeC}!s^}#)V z7zdnA1^cd~xLgnD*mM=!`|qOzqn}VjSuI8Bviq>A$m- zsrMO5e7D63RIY9nR4^Up31?c;lafrR;*}QkgB@3BgOGA+-wIp$tG_yBwpW{6w@3%R zQ+$RTgMQ-l>#j(wIElKtRt$DK&p{dob3n*i8}f4KKPZnVKu5OUM4r!*V3qQ1TvPNM z-=C(0Vvc+o*6uKbMR_Thdb$`6dh1c>SQ7LUL-^yJDt;ZL2_xcdD89rM9oSuiU%#1; zY~XyX{pkiVwc|5HZVE)^mRRJ`y#q{JtOFZXzb2it9{_4bET6) zXNHlArZI5O2OZFvjFCbABckS-C8@RB26}v*fp<(@1mbmiQTXJ;F#M`3<-H}6%wIB+ zT%H>b3g0Hck4`l_|7$L=evd8Om|X*v7Dqw-4tpew8H>|I8swPOy*S~60|>UYg(vH# zfV|URP|b%mAVKjzV)Bo3c9(#t? znBpmLHJ{5be;gx#Eyeuf)^F+NY$N_<*)qD1DnlJUj?@l?u}98g7H5fGoc>;a6byDHWfI%tQ8pSOa-Tn;)wSy z1LVwQEL`@c7A^NRgc{S1fK^$sXzZs6;QPBCx&_gzL$z8nbZ;gn{WlG@M87tz)lK=poZGh}p9&f$Ea;UXzC%EJM zk$gI>gE!81GH}#Cz}wI;gKU4>O9%`1flw<7Mwi|P#n29Rw>A-YppHB;=L;Mk-%YBn znQXJ`aweFRWepQ{ok2%;{RP5}<-`*IV`TOl7w~CjCr@kUD= zCAH6)qmdoQz{Be|fOA|U7a)T|qwziFg5+wqRZ-VBk|f2WWjpKrZ^=1PvCd zfvZKj;NLbOIBA@R8yZun&D&a#o7YjwBvg*noKH|&1VcFQz8CV_nT{8@Z9vwRBJkwT zKC~xpA3k80hBrPRgEeUds=VBSnr3|#g`eC`@o#8Qn!n~zrVGPi&9Odg>KKf&b9cdz zihFqf!|8Z~=~2ACkD>1O*Hgt;$Kv;X8dRS46X@_$64QoP@%gRuKe{2 z{`flxY`TKMV+%;sI^EzsY!H!EdxuERbsfY`&zppgAQf&V)zGB~JJL;G3QlH+Ozrq1gze7vxP&H8$dYPs3y8dW@hO&+1pL3&#TkA_m z3uZP9IhD&(*trF2zsn#?mL(BnfHkrBburNL%;R;bo7m_#+K|2O5x{r7I%F0NP#F?^tK)$ z8x6|Un#qZBH=z4|Jo%?J6}+{GB;)*UkP!=1(Y}R&MB$8|WR>rAqAc(TxY6-|*zDwB zeYr9lc6XJ4?m=^qp&kTvPg?+;rLJJ^(up84F^n#X8lbnBD^fN0&(W<_uP8OAXY_oL z674bFiF)R{jjGmY#_MaAz(IKlu2i$6{W>pG|Ama8jx-&kJof&gZ@=omyT}y!&OHaZ zxKu>1nievw-@8KBEwiGc`y=3A69sywTRFAC<2?1c&x3y5cb}f|A%`08J(srl76^X7 z{f`Wp8NiF%v5TmhXG`*q>%cYbHDK3yIWg(T8L~XtnW#@GxAA!PhZyY^MGX9O0_nfU zk&!#P$)|mWATE7A!5iuZi|;HV7})_Loj-;gVZ0x}v6e)I)iJ_jLOdwgHAJp@p9a)B z>&cw~^}zE?2QPG79I;5}5^mdvkcOfeJkb$Oo|+tqeNzuX;eKD7Qne1Uf&jEhe>7O! zp-F`Gx{=SB0x&c~8!>JNU`Oe7^ife4C$AZfT`wCzX3j|DD0vFc?xw+l1s{o-*9Hh< zJqKv%{|NqEqXI3yO~-j*DyZhNI=<+&n^aiuivH@TQIFHyfw8EC;A>=~kbrZT$~L6p ztY!_f+L6R8OFsGigAF3XE|Fac@$iX$20l1Xi<)d03|>@sfLDm20_Sf-eV?15_FHe1 zM1%lh@G9zTxIyj6O+-CQEWobpXmq`=5>4^Ai>y96VHM3=V1&zdbZf8`S)KEz>J=VR zwi6$t8E3<&=P9-H$l7<5&C3<2&~+C&H{${#udSuV+%^JRMtP#+#*Zn50h0Q9!3r84 z4M16|AJcDaF2hUf`zdtq05<)mOf@PhQ9FF6(694?vE|q&xbV^fs>#Qd+W5~58O9w$ z+K2ja+&g{B>2L*gIm?gAKI#B%^2*_*mO^x7qc26C+k>z1f~e>UJ?PMU3@v};0W}|9 zCVREbP+Q<+oTZ8J9Jd_UwRAjHTMM&YLOvb!(6613zarH z;{F3Qu+E_qX5Ha~18Isx#H4y+NxK73*qj90j`x6!Us*`9;RDhXE+WV3ro$a``r(y~ z>(DbZjr92X0-F10k;?;>pxI@0xWI#i24Wq2`t0c8w(0>YsI(@I@Qsmw`%T>NXfe7a z7Le4nc`)?eLVPnij(TVO8TI$3lRx)YfVeF)k=^wZDe2O;Ag#{@6inY8t`cakGm)K5menm==RpzbK;*)mTuQ*RIB; znSQjAPd)X&+O7kvsioUeB}8eV2&jOFC`r!DoRZKHQBY6>0TC4lEg-!GdqEVHqNrfS z1|o75EC>h*MJ$Mly(1`g#on;%JAupn_3BmM{omvJNMOCLkzZhl?5q6K|?Z;E}m$NKJB{n@-GvLAE@YI>_y)9ok1gTQp$02w=HZ6z)v)v}!cC8^$KkVb)R;0JzE~{__u3@kpXZ)J z0rCm>)ut2}V%`jwtYbsft82NUHFLR{O&6g}+BmGIg%)@hM1N{p)B@aMcL5esc?+q} zbAv`MX|UwZO6;Pe8t$@vC}1DA0MzwX;I*8Mtds5WBRZ+D$I3fs;HYYBf`c8>N!*0r z*iZ@6tAaRXr>?@fvwHZ8TR3s0csEjce29x3eGd+PVxYoR0a)T2ZE(e%h0jvohNo&9 zz$x=Rz+3qm+*e*3WyfTo8MpeQoG}NmRT>uP)uSoIUiScGROyKAW*gwxtkY=wr6ur! z{2sL4=qzwb-w7YweFyhmhy#{i03Qb)AfZYAqq-^RUe7om0Il3_Oev2r@c zYVr3YAK}Huw{f49^y}1eyWkz;6j-)w6P9N34pgq%kBa%q`0(Y;T&HbXxc6ZrBKKlB zJfTkCN9VbiYjnmN8mldVEt3gi@lFNgH{J)XQpg4ymtRMp%oKE-L=8 zyh#&;WOZ;_+c&_(i;8%B!VCJI>`Nm#@Iq z`(Fp+?zF==4{hwIQ2^&?;91moG6)Z!8U>3A!lAeOI$Zv837kb};Q4_qFjRjJ_Y83v zHXZ~-hNmH_ept*MTWyYePkN1vcH6^HB`vh9@*!yVJ_p7=wu4>|UXi2v?xKA3YY4o2 z5!HC?2%+A-iRu;5MtT?J63I#luvpfOSnMDs#W5*lbEX!_q0W(YeaBG= z3)c{lZ84NVq%4)y#HJdG6Yw$ds?_^W@x(2g)o9CxB2wgNN4B*Elk8S|%KG+7YVbrJ zIkT-dmGCGT-xj?eeK;%w^Rx$WHG1~OYtLN9SBwq8SMd6y!f`i1-sN53M!6NH)Vn8G z(&UN^%vPiO!Ar0xrVTpvupGB@^?-pdKcPn(;<*6}8^Q7=9pC`>Al^?PhOf=1Lof3O z+!Kxs$U-X~kCj~p7U!m-#lZ`~gq>z6Nne2at{j2~+pGrng?mx!uJdT(1_|7~+70>C zD4>WB5_p%@4rbMuaEoz|3$2a*f;%QY=RR-}iDTnb-rGZ;gUcaU#&J;stY0 z^o8EI2{E$73^ET7<9>RYhjec&f_Hf-Ao-mv^xvEd^Dj@pL~Dn^0~*E1W(k8BR0 zxBBCsIPs{rKbt((mJRG%Ey32b3$VCtIx_NC!j4{h3%5;lKqk5j+-yl03K+=7U%tD9 zzv3>$8obsJr3VY(wSmXr+iGjLxSd7J^?nG)q`G2R%NG+F!4L7`lv+&p`cQPSM+O=k zkpqhZ72tZstGs~l`TX-&)2NH@?f5yvoOlZH(fn)eX}lW|QB?j#EgmagM2+5;ff zNO_eO@nm;8@s^pDP%Ev+^Ir9fGuN{-ci#Jd5>rc;l8_Bw35J_zkn>QY$0b zxZK*MywF=_yy{xOn|1aH&!{knuXPXO*^YR{d&pzrc@6D2GxP$m*=qw=Cbpvi-ZjMJ zVLs^L0ymVk#uXN?re8PRl#jVh$^-MJ`Jp;dK5nyD1I>-dgFOBrJW<&Mj(OjgIDfwt zs1FE&2L05K>=cR^BTht0rC#8G`dHkwu@oQGqYCo(;x`b2y`BmUG@jCUBM0GC}zP%>x>c7_#v z7QB_|%zaQ&4R*OP@Cl~t@dtKUAcMGxT*fz`2I@JHHgFs4AD4;gGR)zQg3-v{PzP)L z%z?A53Q>=Typ2yLQ;Iu}@S;=X zdBj^I%GEiMYB-inI`006PRpm0nhLVKr`ap02IJn;m4-5EWKlhjFgr*#reyML(oA{j zU+j3(`HH04`)D2^cY};~-Hr#CHBnlpGbr=xODLAH3@^d zzGYzij9UEoC|9^eya<0P;}5h9-vHtGtFZK*CJ{8;5NQ>z1*@I4kPKcssb^<2mU zxerbQ6G19S!oNVJ*}2??Z3woo7lZMOjM0>4Cm87$gGO>LLZv}NICu3-v9?G)+;7O_ z=H0$B_*yGOTF0-Vb5n1E6?==go4*Xg59GFS$K`Co!~1)J8Aq03+j=g9Tkd7z>SM=a z!|TOh$cjQ#f%+gtK~H3wCj;7D70?UizMw643piBVj53~kfub{av58M}z`9}?`n^{b zu-+;a$!{%CGgx?M|@Bj0~|MO0^26vhu0n5ag)|) zaB6cRS4R+yZ@y9p4$R8LJuc)Rm9+-=!7?72c~lX1JiHr)DQ<-+4@ZC-=SHAh8wN@_ zlZA{k=cCq|t01Xj4K!FZ0Ay{zIMm~DT&F4WFiXD{gs{u7)6WJ$jrn`g#?~Vo+x7F& zb$KS#Gm8Vy^IfrCnTg=jiOFo^je9T!Zadd<;zl_8YAjk%t_z!T=AvHyIF?(v8%%RO zgl;sI!HK==xtHnt4)0}aqA7|paQMkS=*u)Uw!uU_uDa?Q%)b}L%`wu1vre;-Z1e_{ zR>S1lY~(}ttv2W^YZ^?;)RVn6rf$oH_coS7v2hRTv{f2)++-<{Vd+fmd+~|v*gKihTkJ)iJfcb1C@7J` zV=ofQlefXPEI#3){FK}voJInZ#YFpVU2;N80X6W#2trmBqiWZ+kS4_^$;SFPV%wMq zD(&1JV&;~Y=>4-=a#^J-S+T{1bh~trOnDYeiCHn^x&&iNzpes}@EQnS#&SV(fBJR2 z{Tj$=Yz^q}c!a77G@-}ITsW)2i>vu{@hXE zRbU@Dv9%0Kf7OBoB~@ULPOgQoMxW$f-lGnECF#hKk%;jc2IC!z8Sn*of=)qiY-IZ$ z$m>`FCYz_hdEs)PZ36{s=QzPQ>!&axZvi9*pTV?lYH}mCE(8^dY)-@bQygo5S@_&= z1?+L;IOgLigUuT6gc=Xma9mbbBBjJ*oQa8jvDBk3fVr*+<@mL59kYgEXSpw+egAo| zp~ex0zR$q;9ev;honBmL(|Z`>T@7aI@ED}t#F0l0NL1arg| z0Hu@LxvL(zu*YO+fOBE)pek@OwnyO%w&JxXe0iw>Ecjz4`;wIwtjO2EymMz_WlEbd zhmLtz%BlX?4gYHHfygTOv3)&wj#6RKAq>Q2WWo@P1ap?QfxCmIlf=$Z)S|}|2$w}{ z$~G>UyjAZ`4Ugzcj@y?HZzcC83okB7ZHZaBmvvIb$}Vplv~#=2uTG-t|M2Bxk*k7RUx*gUM3f47!bGJhTwAkRixmpJ}EP#FWFkZlRU++qGrVy zlLJ@2A*U@K4lVQ=Va~$!+*L_c@a61cxKm3VS)c6-A1}HK*h7wBZukf8)6p;4&LR=; z8l?}WNvZ)*nGY7y?{`o=I0l4YK8}UeNI0cK4x9e+9JiojIV^fJ z7s%Jn16ZFOD00Uz%rR#Q(7C6CvU^SDDC^pxAWsBst%8vY+RRxq{w>%UcNGO3G)D3- z=5afCFVR8WXteCk1i(5s2_56dah38(%w+a+?9hf#?gMN-JQNg(K1{uYYDbh{&WU&6 z_0`*u*3fnEcz*#Jn5zTxJa2#l=hwmNA|bBu!5zkoSiv2-;2z}l$i<8gT!NFIokAsj z)L?_jAjIq7p{LDS6xNKa|da_N+lPpDI^9T>qqviJ4N-7^CRYGO`ys&PmqoaWT-cV$;A15XR7>MHSzSA zHwsAGPr6;1O{%1EN#FkGNqg03s;0+0a@lNMs=bth%fFRJ>vkoASL=OIOSKce%Sjc^ z-yVvW9`(cGJ}}WFhAI|6cP!V}ZZ#aex(z26T)QFx6FgSZKy7xY1xOXKn3tEX&{uxO4Xm=YU}; zn2>Z2d$MmKNN^`P$#aF6-jFs}n%xQ<2Rt<^DvE)RN9RFzP6(E4D1?5QM%-JYl(8pu zcG$yVuxncjvFZ;6AhXsF zEm+YDaF3~QP3o6}oP-vR#*7JY>+~#m`}j6s8~+YWOR|UA9%^K0awVyqUxE*CODC@! zV3Sv`Z6Ot+;)x@JH^HzV2clv>B4)PQgK5|7@ap(JWJzfqacS8ZJk~ymP|!b3mZ*f_ zS5Kvqats-=+TWObbbdR&XNizZaxTJaOLI|EkS1BcNhkOjB4Y1^Cq(0i<>dIhd}4@i zC}~*CN8u)qLGHmRSWvA798>Cylm{>3GUrZ2z8PNJkjT5h*s+qEI^C5MvpE@9ixpsl zdj{alN&}(Ek!T027usyB2|Dyt;2FI$9BYfY+?|8zVqI_-s`lQ?Z5g791((v#jbEwc zR$55VG2M&6&)ZImo8@F4ZZU$ne!{e_N+}-qk=hBt{-`l#fA^7WLK<`tWs7IH<%0& zkG^Uo4m+GL88zonm4lIiWlKFg1ejV#9Y+#cQV?w!CfWTUF8LT-CV` z!Pb{QkFA13-dk0A9Q^q$uQK@D3aj{xi3`{}l?F3+mD*fOrB$nlu1AW6A4WU6mXR?yy>M`Jft z9f}iK?@VNgw=SP)<#vC%Sg-Y@#1qp(!SiX(tKttVJoFqtu(5aa@(4U zc-GfQ1(^f*=Pjwc`WHJ1XV<0pHU164SOz0D)yV=zk`{T;-IyrTzJ%81sF2StV-$1X zA>!UV3>SLo@Zw!d@R8i3#LA1M=!kI}u}YGMZ&5QvgQCoMGX6X&(yo}?zy3A3xLtv| zwoQqwcs7>I(+#)e_2^;M@>#)xA1ATYl|Nx|NR?^Td`;6bY>S@-dw7<`398QAS3{Qf zIc|q}l#_wwdEG{f)oz0=9Ou+pP<{_Ad+?r_w>pU}{ThMg`3L@%7xPA$U#Tpzyh=SV zAAHW8Cw|jnp>v?j;^Uqa3(d(DWSn&kX*5NRGP5(lvg~zg7v&~p$^YMd5n2_ zM=2u}h`hPUi<|uG2`2pXSp}3`;0p6& z8m^RYx-+jLb|vqz%WFQ%x02ssex1)NV_VEl%rk#2%%?OQF7u4y3VDk5TGaGsODLaF z*X*3sS6JV+Ewz`CF&Zk<`C~M6zZvU~ydBJSa!g%@EK4gYaJn!-Fg08>Suj-?9yVE& zh#_WAdPb>ph92lLdf#Bs|7{FRMjuuW`r5__qv!|Tm!%LC9@v%aK}0M*|_g(Y-j_J%V zpH9R;mJ*$yxJb%!dRHehSq#Wh6+{UG6Jo++!svY<5;D~oNNN-lBAg}& z2@e+0QE)m+bZoRRMwAqtNE;Be!4zRsY%nbjw76(7NS3-FDK4aQmxzRMf@x8Kow$25 zH5n93Em4>-MH>H4*`Obs9?ib*nc3;{lPM{zO>#^ogU9O8xhE58Ch~vAMDwqhXz?`@ zEq}#CtM8fkU&~f6#=q4mQ-;R>ifq>1Y><-1rh6bIjcxZpN*cTFK{wfkbf1v=*>?}5 zenYzlQa^|8K{wfMNhNXEw~{#gpCr-opC!?W^-Yx;@r@*o{B23pW;p*y)>RoUe~CF= zp=cuY`%5{sXo*x$5C$hDB+xt;{H3RcU+Jlof2=O~lxiw1pLi!_e{@c2n=Mt!Q9tA7 z=&$(c`ZYht{EDA$-}CdoE=j!@|F#C1GW36&pK3IL+`9>c=rl=BjQ#qAFUy;$>OIuM zd8or^M}hay(c>Ihy+VXBi9*w`$D2j&CUjk`E(G9@#ueJ@MUxgL0CVUsx@NZ%N z&RqSk7QlbCfG5lKy9G4=wm@fUz6whGDrom+k?m?gFGz z4nG2~{8x?CM}PIR@TV zrJWrOR@&LoDF4&h(K(VD(#eq;(pixj(&>;I`gI%rJ^elDNln_ZGe>`){Mq;?_4_AN z(ukdM>Hjmi6nrI@0bk2y;IHH|=zF>Rugg_0#=oskri_ulEtfyb_21_#=qeoibrn`& zgnV65l^H^oq9|4r7{yd(gt8QaW0PV;omY4y?HTs1=j3la!~f$M@uw%<8b;EsVQ7pX za%x~gSl1sNVXVOKsNZ*JNMJPVlELq_5dUf?{6`jlsapTM(3H+Kes`f?Dp@=$@;gMP zKSuPO4F4D#`Kvwt&r5!X?KhXCezl~KCHM}d{*O_1)`A}+~ux%e)OHui63vw zUp6d%mnG>~+9M0Q3Q=fx+1LD=T>WD`?$RuM>aS<8;GfSA(9Kh)oc#*$kBWSk`u#X) ir!f91s1#Co7o>=!j+5ALv#eU zOut8S@{dzzSw^G(yUe}*mEz}P3#g9SW~%e{JDEBujobWgr}E{7%(X_8_Xqbv!TFQ8 z@x?vVU3DL;r0%s?%{hgdq7!&fr6!d!oy?D~y9pyMH=^N*Ca^y4#hZ7Gpz5<1;rNm%e8=xp z7&|r;g5N9f>*zuaHfy20_FF-^kvq(Z_u*}-GCW?-ivJ0{fng`x1rxrH;NL3C!DF#5 zic7R$>xQo+xc?J6#~;H}mR0b3+fGm!WUF9ySxcKo_r67CYtzy}gm+=w4QKgARr z`PzlMtHoo|fHn<2(1}0e{dl#>SpIOF0$u)aIJ*8@gKL8qL1M;!;(MY9<_K2c#X}EZ zVM!I#dCjLm)w)#Asu%MAQ>6FElW|PKy6kno0+YJ-O$@F_=|n&PS-)@~{6U@$SLt^w;SE+@HH2lJD1on#eFr z5ROE@+8fOAv;+0eyC+PN^1!QWK9a#aFVqJN?-ed-tt>-x{&%k#eQ_32YI z+Abh^;TOTna2~sI-G>`>nbE3TUG(b^=jHFM>7`NK&Tkj$bC0n}WPRT$h`%*}Cd*vt zP7^D>U$p_fpUdz&f9&b^5C+?3DDr;HfDI>&aO_4mzG>YVxcF}}ZxAa0ozH%B@p5Gv zKh**bg(`8(Q{;is7a=zG7`QFIPhyuYq2uQ4#k=P_@rhSFko1di;D#lCorTcNh2-Y$ zFNjOExX3qWtX%}qcR`){t-TB`2S-uW(P}(S`wV{6m7@FhyhM{rzfpvIhVi3g(D;cK zZogy7fA(0>67h{d&R63I`#kX7;sLEnY3#V(B(y?bT9>z!TC`A@5mJgildE8PavnA} zjpnr_xZ*m1xJLLFmnO-s}CXl|(%0#gV zqo}z4Sn|TpfmVLZUcF$mg;yRvjn}fXgsKmop!*0v zp53Ma5B@o${b~#P@#q~vxRQsFkWK+8;8dTJG1NIcyUB{J91s6Fm8zD;!!Xy{OB=Zz#w>NZXi+7w#lN&xAH- zt6%5I*+a4<#kv$K0;-67mJY3UHRbOt?dWmYF5Gyl73at|v4vVHV6f@JkbX3h_KD9> z5aEM8JEn2F;$0AVeJtO)!j8M>`SH=uvrtam8a;M6qoMQ&dfd4e3+AZPYZ8{+b-4@; z^^xX^C++B+?bhJiG!=5{j^cBtL9qIxNBe3N>GczO)HZtr?YR_0?=G)^H-@QD_o_s& z+h#H`UU3`GFG>YP#Vo+e1Ei|ioKwwgXU(7v(&yQQ6CMZhyptX@<&7*)oqR;ldCZ6h zeHq84B~-c9$lLhG<`SeWY!()ppCaE~-U~i$Q{+dd#o>Cl36xBkNW)$`Vn*`_Hgbg* zfBb2guvT1-^8ig&+j0U^FPflEUjiGqtORrFLvd;SW1KwNfq>3xemC<2lscQk$y=wv zOm9CrWM*UO>li3dIRU?!fLq&!35!xDqj7I5NT%%tx%3)Pl+%Yzy1z+h?N1g`dlU0N zD8Qm;?hsIt4C1k^VDd_dY_`n6P!TKKCEbfk6P`0Q_p>-9xDcmJwWEGrMfjjX0JjS% z*7VmAuc&N6>Job{6{1C_NDg4$qGF5)N2rW;!Og+R_*GensohORwA|2I77jPu$X)hc8TT zLyOT*p|vRGZal%;*f1{8k4syYijpY>7cLZ!gH0dR(wfCKw}4>p=|LFrt2` zpy+ES9Gx|sbUBR1PZ3W1R^f3$XFH0q^!>y=m%MHQfy(b~K`3(C1j>Pr7 zcHC|4RZ``mLlVBa;;zy{9Pw}}pL0$W)+_1Jtg#ZHk(@}PUwp&c^Ueyt8WuV4DJ{Wm zN=x9|m^H%H(Vpmkz8^l!p25G4OMz`!XUIJ77BDkw0p;@+cz+j#6T=E&rQ2LMHeM1w zukK<-A4~9*fjC;lW*`PLcyiE-{_!{>)KnXQib_%FB`evpI2mYMH4J7QOv4aK1DYl! z&$k8r6GRn!MTz61u%|bJ2(P=buXzixY~@8Pxw;;tx0nEYScKJ^C*W~sh z_!yd+u9F@|V~7}di_H(^p&_spPo_kIkMu{jQ$CswEUADJIZ-xi8(`V)By@_E$EG|^ zeiX%%u({`1q-i!RJ8A-UR}$gdl4fSNuN&{r$;YE&4LIwhKKuyKZ9DL1siaLr6ws2&BA##^DgmAmDqZ?W?jQzLlNBM(p8^ti>;$+YW*HYP4D!(A^Wxr5tE z6#qIO-r8BYR z&^$Cxfwvv=V2VgFIlb>0Cf{$vV(mEGnkNNU{~W=xfmTv}t%`KV2QmHG51GG&3QbyS zPd&fJK%;d9{`-1-=vySRNhe2R`N+FyKhupL(-*+^hW5(HLPx&pNeKp*Gz&jYJ40q& ziRA_LZP2?K*V+nSy?u zFG+111*@PxV7_Jqc8ePEk4jTv=Gw)OR%M9Pp!sL`Qo%YFb29CM#7GtO4Ks27RHLcX4#fAVE>S3J(A*u z)AbDK(P0mXXy#&?-Mog;qffx2sug4`r$d4#fv*c?ux~*pEU{RGmNW08lHp9YGg)Di-9=u0k2Z^%$WR4Qd~>`R4|TI~Nw9%Zc%rA@P=ci->Wq`Yc6j$Cu-=!Q;Z^ zJAT04o9D>l3#UQj(p&gkd6L{~Jd2J;UxB~J8)0*<8-%;|fXWId{O{ikSS6pyb`1HM zL-q|g_|Sp6S!{!n3K?p$egW~gX91OWx&#$zXK>U7D^xu73jKz+!|t>-EYL3<#_0fm zea{k2bio zo!$mpN7iFD&4GBEQ;>cCfS`P)g7B3<09ttm@fI_ppVqD9KUMyKO@TJ;&lzBj{#oR& z^ikm^sUj8;VgxZKL{Y*Z1an%h$J_H_%XAN=5a`*I)1CwIW! zlhMp{wlSEek79!s;h;&>X}W&2VEc@TT%a)uxbH`S@zfV2`JOx5Gf*wO8Y~6(Rq9bK zxQ4AHu9e^HCi2vjYos?b4Hd5G@a{1M2d3yu%J|x$J@r>T@PuDIC~r7a(>~x<#t%Qyn^jYKPh2 z{p7>LM7UqQ62&Ah6Qjvmyh~$mWy;(SuyMjEtm?f6%dW4aJGxWZmZ|RaMq&XWPaM&; zUIu)2SAnHZ66D3p^7p&-X{VbDI$xIHTe5C~Pw-_duX-x1DL28r&o{%;8Yh}FEQ1Ur z8$-Qe9A@dI;G5`hGWLf&owF?oBb;L^V-mJtp7ADh=`_GbiGi#6?{nBDmqB!HiHB!f zZNcuuG5Fy$6JwhjS**7bJbTwJup;SfQ%4&_Ec}Ga1FeOPdyYVw=0udVQb5)82XOS^ zF_)AjK5*<9R`7LQQ| z43Cdjq?)r2UR4#>0f!YqIKHAuxb}!X9GgF%57viJn+twuzbzS8xZD$jWUav34fE*e+cyW`F=j)@N+X>X!_i@geV;DEAnwW-k-kK6-=7nib@w;u*XVSI91(9Y85t zSC}{2gfBX60>wvvlUScB2-N%oB3re1@1|_nwf7&f*mW0XSSP{>?-1H4Iu{FE40xd0 zQ(~4L4*kX($+g-)FfAbzXBA`t8}^Z8IVM9Ssb-dCo?vq-5A%Jl;FdAb=(j5r+scRY zgKjo(+(HpPP2D9twc<1JvM+!O8)INx%`Nb?R79QI|QY+jkK4t(=V5^ znT2jc z&}y6miNzVz7l|lI<4_;+7ncX+= zr;AQlqWbSLJiA$pZTojfcqQaJ#%Jc@ZMB1Vu(+5Nzp4iFO%w3xW=-lkV3!X!Z2TWA6!Q)AeE_C zULXp3BOx$Yf{xK14nj#Iy6Z;)xDKv_!1Q`t_O6fp+k6ks?!1l3)zeVZt`XiO>4Eg7 zi|}{4BDXs9kmN_lLx-)UaN_Q09NB2em+P1d#HQJSZ(soF3wnlOQ*|M^K!HCi&SPyB zPb=H>|H1L?zuAM?*{tW#GTbm(0$i_Vu>ET8IO+0Q@<_9oO?_m_zua)c@aHY~IJ5~> zFc)wBTY=i0E}&^4!)s(Rpkwa`P@i!EY=Tpz;Esitah~mblnd8~Rl6AaZrBZMj})_8lfhC`b?8^0 zWPHAIwm@PiYclNcp;?I=!BO1}-CuTM=cFyn>!N_XTXz)~n?z9cAD`j=VoI|>eVZNp*^!OSCf+1Mi(vf> zQ+Ak_vG&vl!a0_oNX|4fCTnbkt2dhBsio1(sQnFGdNz{{8(q(i`fG8SkzG)$SC6xY zKVd7bq~ShUDF}rTaAVypm|V1zrUdCh>ZZN8^6@J+%BGr3PpBugHSGeElw9y#pM+aJ z^?_B$WZ=C&Q6ZpN*rL}AMaOHg+AvedLNqb(!7v;-t`P4R=<^L%5*YU6lG8TFG33b! z+%4x%(2`?O$RRw`+J?@@WKntfZ}Q|+Xr;`>I(MQtb;fMCFstQVZ}=X0%-+%Tr!IwUbu$C ztDeA>BMLaC;j_~o;S<5;8`&@=@dDfb%#b;OBwDA1z{pjX@!&EwOzmCB1O4WZ0rN27 z-+?K(`BD`7&p&}g9$myfYmI`9*J{ySYYBwYS&$dE9#7RM!nmOztaqm&-5427EMjM4 zfZcK&@7e`kd(7xA@e6DmHGyrd-gsVVbX+7HHLV(<_WHQ z|4}&qPlG@+bsuz%+=r)q$I&*UF8F%a4Ccs7@{k>`D?{tIU=}$fd|6kF4o0T*S;STL z>4OgX|2c%e;`d|K*7I;lFW&BL{)5=?3rUw@mv*zwm|B2e?p5slA&XaqoPKPSUB6_-hRJT)PN# z_szw3D>SIaxMWiNK8^|omXO7#5>ef1H>1YdSW~%ncrZl z-_>~DEFYefUB%1;gYkgRN@14g3*tMc0C##D;t{h%kbl}h&W@YLd_J53muG<;+{$YoP+)yyxPc zXdPNvGl|uH+Qd?Ri}2;U&kGZh(goord&gqOzLgkKAelk#q3a%9Xouxm)c@uo5O{Bk~+i@8Et z89`agCG^qLHgafQG=9+6Mf;}*n1_xTjX9G{>ZY1;CcB;_#dSllkq-CK*vr}{44u`a zesl=$g;)KLaA3kn@YT*nz4LeAoU$%euK0)IDU*19W&}8|aYV^4I|R|aHz2vjo!pdq zfI08q2|V_mV((__vjOd9L0ID|@Z7W$_72VSt0@V%B3uL09E$NrKp*BWav&0R8gQ+7L;AcY0n^1Lpyo+G z{0>TmLsM@OR@f<&39Q2Lt3-I!!P5}*Vya-@_Cj(Y@fduq{SV)T=*T=s>O(>I z?qnDU{DlVk>tX5(6PCYg6gNI|4K{^56ZBp6#g5?N)V3m=&AHQpYi#rIblwd9OJog; z{AW%VM@d2B?ikp3T@F_Vd!mbOI?4?a&?=NC15dM@qwoG?)ds_O-j`iu^tN>-JsxJ~CKNf453`5-?wqVPMwAdE(~apk&nkG`Rt}R~#zDM{P_U}h ziQj061$ucUR9<*Nyss_6u7tCoFJ;f325Es|SUc0To-w3fqP(~FDooFAVG8C_e1yL| z?$j$A^0mo0GjKoZcGtlVx4jUtsEMUKxCrOQxWKF{1GuM3jNWlxheUTjj!|~uZqbuL zWvv1j2Za#Zs8)!$D?#75L}Nwa74#Xmlzp7XaJk8QwoE39w0vrU!dd0GW!haZaGlD} z$xvRC+{ThbXX2NsQZz(;KEB+wA2Py+@v^U7=ysuq*-IR74)>ged6qjs*X0r`UNZu8 z|GHysbs#o{6=BK0>zIEc0P|P+gP4Z6@Wz4ReE8#EIHGkLPT3?uO`j(TpGg6AXzf9> z!x=a&UY`uQp2d~tq|kMPISnarMAP4+xtCu*d@C`>a}keNTjybZQ+*Mnj5Gxe|0ptf zy##$K7sVp(x1vRBB;0wB4^_J*>1d1j=(?@~R5jDc59djE$(dJ9cI+VC^K(crX~)f_ z$8ng|3gWilJyb7n0Ufx=EQa>g%zDK7+5kKxwF!lrexTR=By6i=OtM3ckNRx@om0-? zealZ^I4+ctmlCvJ!vvnUHVKm#q_ES*UeM!}2aUqv>=lk<6}^LG1#5vmktHx z*&{H8Y=aNqW%;=FGce3dtvDL0nCpZuLHeQ0H8P{;V)>!s3 z&ICQ=Re9+Q3E{Mv`{2P8MR>1Ti0=&wVbzZD^zs`EJnC?QOwxXY-oCcZPu?w~>&4=+ zO0rKlGSdMpZ1$n;&7+lXTa>8QZCem2u4S_lRj5VpVeq^;hJ9F>3)va-xY!R7{`dKH z>{j22kK+zQ-!5mlyaB9}Fe zLxsg5!Pp-i@Z?YvahfRuJO1r~TDlkQ1wHUo#sCuZhWd^N+3;5{PhfvWnsn9#;jeuq zIBmWro#&qG>?GY^x_SvE4fcLuH_`Q@iTi=`y3^L;@KFZBDlI^AKqV`36tl1 zgt1s5G*w}sCV+>7Tp)Zn-RWiG$g9%Rx>q2p6Dbh_sZbzxapo=k<4j$9^FJalPsQXL*# z`5brdG66Ts^vd$3uh`suSD9z84FugQz~R#_vmbtEVUWpD^{Qc5LcfFT?bptizn%fI z`UE?);RbBG>_iI-Y-sq1a*U~$q3x{^xN`Sj+>=)#G`l*MN(?6BLpeM6PdR}!F$)|s z{61zD1=E`52za(O0wQG*zXX3H6LOSkK!mkWB0!OM=c#eE(WdaNbSKVx^qHk4Ri`mj}(lqf{dsou*&u{lQkIwAWA7LZgN6FIJMBGES)MDoG}8(ZnhLcH)>* z*D+{q3;e72icCKv>gt;Cju5S6TJgb%6dHj!sNaT>{ z1#>|tW{iz(TXDs0ReB`17_*-JzdKOIyS6t_#byL7$@v8fURc0jY@N_MVHkaKUzv~8 zHK21eQ(@wDvYks?+$9y44{4XbKEBpMicSfJ{^ffwEeA zt~O^FuD_#B!-C4;XW=|J+3^!zy4?e9mqL&hjw1FIKgq}pH}2hUNG{19LiM6;SZdQP z7;g2M-TQHl>1vB|6{}fTUDgE;H0+#pV+C_0K`XYWg`>_>aS!*zf6+*7_ zJ~HoHy-@1sWTZ3H_=O>sBHm>Q8oi2vItLE%ZfRtAm^ClOR9N)g9;@zKLgzU-{%ZCn z5?6i(ekfkYXZkI8=MljZovtMF+##q*7{Hm12LxtXQ|T&P&W=tQ!R|f}!D~LD5ZK^@ z(Ho@bqwY7t_Fcu;ievDZ{4P|@a_8GOcncRi*o!YSJ6V}rC=CmW1|ed|*MQ*ZnS4taB-v@Z_gZU;KIKAp|~c^lH+%0?`_s0}VTdUX7_U6{RB6qouX zlWe;Ul|HLQsJq`vFbTVX^FK(_y2eJ5SW-z=rqz%GlTzTuQ)L^SLe@rFGn5;Q0 zf=x>r;8SuG`ES)G+#FUY@Uv`$NJB+VOm|lknY?fx9krJ z!$>+>v)wTHCgS(O2~2#`S!}o)#fEjyfD`T$KyIxp&rFe{84h!()TNC-fNlXX)}(tW=hRnK51@FkKS7w~3+p%*{|aGXj6_+>IAI z?fH`V8vLoBg`m`Y1d;O{Al=6v3qsLIQ6s5`teUROrH1<4^?gTJ zg{v=wu6m5m+U+1a_O>u{+j5*XY8YHyJ%*Ajb)x;lmS*d#;kN0vY}&$noN;&+Xt$ro z{n}Be7IY43k4nSt5#h)ltN|naOK?9nl!(8Uf-U2V(bYH+f~J`8oeLgg_?t&8LZg=j zxBVc8td2t3?yETJrAs1D>|AXO&wo*6C`DDT7kU3dn{HMA zjJiYF#5#$;cpx$k1AkwKd$RRV{l1@s8}Gtx6U(9e(_9oeIGRV<)WaON*=&3KEGShN zMYoLF#5}xDV%RV#{#!YiTg+>N2UlM}SzDgaqP?G-I6ImdEWOGkM7NL$yK?cCdly8{ zdBSX;y#Pf#3kMqJKxOq^?8;lj`fdlI_O8Ep?ByomB}D}qyYw|KGmFNR&#G{hs0Y2J z;fyug?jbuo#EPuliSG{TQybf2qPul6XuO;~l<8a>$^yJ-W5oiRpsJ0M3rw(Xc?`>( z-U{}EvzJP7JA}>>=3;7+Y+<2-=Xq>10-Pf63;P?wA*9qw zAiwS>3bQk~=$9zGspd|njc$fhU)$iQxf5S?cmrfU3?+YCdfD|?56Hpi#_*%d7<#X% zLG+Fz%%jH$)1y?m&yN@=`+5c);>`GyW7$IAM~fiBPl7)a{Q+pboD82iiV74`AYoM` zxcXdz2IW*xN;M{Y$2Ppc%SrHd5A=vEXLo~63eUc)g$bYH*wrm@STNK}zg80i=f?_k z^7bB(tec0%V^+ebxe_$__9ZA?y#>occH@6<>x7$w@8FhzL#Tzj*yx{|p|mp(WM}>) z7sx+!3{wGVK^*=|v!T%y#e!=NZ?NptB24|So`t=-0ypF)&;w<6Ah{s>Hcf*u<9EK zQ*+!+>_e^4Y&IH%4-+){09RarwzrbX71=Kx?C$X2-2wlF|LH}h*Uc0jZ zI_HOwv-x4LZRQPdU6qOkN+R%aeiQDzB?0i@B)ZJd5JKEp|oG zeRk<2c=~L3JNSaB?OF&nhV8gu`g52+jJlig_^>K{}} zKx2gD5NGJfM=z4Y@TE<{eS5U=gUUTYe_Hc79mG>%8BCJD1+BNOXpTl5 z>AhVE#Qy`i`o{$tG>YNY0!dmHAVuYOf5XylQ&N|?TWz9|!yYpYr zw|@d(uj+_itJ*NO(+Gw~t!EYwx=8;^H$MK~2qEkP%mp?VXYkX!aPx z%9r4>{a5kbAA6K^QRCO{>>wJ!t?a$#b}YMK1|iFwh)aW_~O1)>v)fOqCSo?DH@d*~pOP&sQ zu1L{M6Kq(sV>O!?;6tYW*bf~lJ=kr>*!cMunf1H|tezPG#hR}L;puPD^xAJ`b^8>! zo{1$+yC1`ezJq9d%#qr(1;B$jpV-KcPXxCf`tsQ#FVX4dOm60L8ryoO@R*Q7Hf)^* zw~O(|Ah#M;0R*`)1Ye3&V_2;tCR%Gy?eOzNx=Rria&O?L={r!{Ef6j?7C@xcNu1<= zjWqu&Kut+WYL~x?b{ucR25TicO7Si zjkx8|Ei_AgOww*f^B~FVf&*f=$iFdL;j!;a{F52L&-6aP03}VhUt)`ID}+qv?OWFF zSq;_acHlToXJ&KiHq-bNLf$^jB_Z>-z^o(Duyw+6u(J?YLqyoEq%KMb@Vyh*zb$>a#x7z*cG(fdO7r0 z*bD7DBgv9DMIL)bAE$p;rn$SL$*PPh^6ckADt)&X)-93c=YIC${FVXYbJ`lFDn#Jf zp?56#Cp5?`=-w>=CDXDE!lk0F6k2rsS!H~s;dy%S)=o5#%b(gEz%@MR}9%cH?Q zYi{m&8yn~r43!^`cc*;>i@afct?(PHaea=5CU=lQsK!^h7jf0DLqj}sD><-f4Ww;q zggFTgaQxv^D7^3sN1_z%kqrl#(HhL-Sw3zy2*tquUl=uZ5oC^fjn?<;u)Z@6)P2*~ z@K_;yp4toJR%r8@ce8QeODry$k%m3t*O-&j0NyWpMMi3##y|Pqba&Eo!M|27GMSA8 zuhTE!+wVj?Dl(g@f6HeIVF4KAc?kX;*oSYs2U+2;2>Nc{Q&6@%$_Dq@VVTZ(;fTTW z=;>a~7T;{$`)&b@p%esl*YR z#*W74KShMuc3%8No5WB*YzlHs$5_g>qmUvO2yaD;F)}I^{<wiQlRs#Bp zx#*pF9HYO-33YN#VpHp3vNk~p&emmv&XG#;bEgHYI@t_Z{17h}ok8;Wz z1<7OeAwB&~kEgw1!3VN25$=%*?F!Os2V*taShd}rPi_7D3*qHXFR zDRoHO3zFc8!`;f%F$*ELX9bE%EAv@bj>CctnK;`&5NZc&NLg72QT%I$wwaUBJvE=m zPgw^u=eFVB2^P4v$sFg|r$YA@Lz<^ljmjzqNXT+u(0zWIy%)Z~9q&fd*(M$MhaE)m zqW8kRm%7ktQzFrrox?9bv1FM)?&7f_=6BY(Ntjz%4rh-WvyFM%f&Ujfl(D80<5p9$ zAlwUl@*Rcp&ySEttKzXT%MnWa2)tkT9gW&Y!Hg;Hw7q&Ou9VC$Ce;g#Uv%K9ra?xm7K26q3Apjm3D-w2pxahm#SJ?ZAeCrvZ{HunK*2^V8q~rd zF*O+eGz#>6o3ZMy4(P~sNonEy;F1V>KPDtuK+M zj?wV?L@DT1$xu8xR(71Hva%43G!lmmZKVsco~Bu7OTRE5x22Zrxv!X%Rx7t z?HJ}2O;pb|V@GEi7+k#~ocr)O{L~-Cim50K9TKYO1BY48sSEhZ&5~A^r-5GL7f{+W zhseF01qWi~=<4vJxX9)g#{8WIi}zNMy39ndIwirzIgMd&W41v^Nid8H8^JWb=d;^o z?y&#AvqIuCg{JdJ2zAwi84309;Lq{Od#w}k+MFF+MAw#n?v%mlM33)IYQwQ>&0$q< zFzG1O#(8TVGwtq1l+#&*`zj|2MjSp0Q!STa(es;V<8up*iXzZG(i^Y6*o4)pL9k+v z1qSI>vrRsd+&v&0W|2~6y73k~$(;tj&pI%5^RxO)MByXHscD1*!GW=aokHd#}*i{lkJ=>~& zW>vS8N!mpV&&uCp)+!&Fzwk9)elCW$L#l}N$}4bq_aFwC?88~bDrDn{cMv}L7KTb{ z&~iyx2nx~Wu6jEK z8XNKD^Bi*LP7~3;8$um6G5q&OgnhlT9zI{%4Qclpp+T=o_&}@~sMH@;;8O#)X8mKo z@{T}|+$$)K^nf@o1Nxu!R5s7J5m|mDIM1#_F)-lMro2X#_1V~zsRE0JX13aD4O~7Z z0TNE$X90yTNbL!K-Wgbfi@R5niWR0@m}`pGOM_8%p`G)^BPzlTk<-yL!4$)e$k6+h z7PQ!JXs%Vgfu#cn`0!0)_%ukCS7-clQlA;ijIQ;tr*RbTS}h~f8h4=bb{Dw3CKnUF zx4?wzVVI$4%h&8qgg*09s9s4i;fMxKoY=*3oo7Sw$%SzG>`y^~-D&*VC`MQMufnPb zNjRmEM80=K;;mn4zqu$p#V>3@~9#Eeu~#f@hXCV&k+c&|SI@HqD;n zbVyzw`qIYGP`)4K&-s#^^P531&jA(h#V~d6tyr;KlTk5b_8YG` zcwUqST}wOI)z|>DMfZY-jx}yxe22(eiea)+KHD?819y$lg~QqxU{vR6mi zm)3-X+r-J3wj!A+nQQ``X%+Y~Gy&JkO$1xF0NVdmz;cqe!sTmHbfvQ#9quxRCQlCK z(mm$9zoHAKNH~z=6<#Rnc>wM%zAqRNrApH*GT_3ICdiquLI>U-f{b~L$c#0>{s-4U zqF5RhEOH{DL*91Ng9UJGpaN8Mu0z@5=@1reNgIF7h3ZP~=mH`~Vf5!z+*0arO{;xMP_LwkPixx1Hu#WM^SHDQjk z_Yh0p8rZ(57?!XHc(P+E4|rw98>7|_`LZu8-R2Nmn@3oqULM)I-WMKc=fpm!nO&ns>jh1Wh+-1s0ZY_YZR(D{{P;TAg>;_52gw9pIfCE`u z1-NA_J$ytT&B8~rGh0f8yUSwXwcH=_(6&l2Zi*(4EGmYk3ni>isS)dMM$@tN7jT&F zF`{pKlsFId-Rrif)1zs_>8*+w$Q^S6L#ZsDygkG*FLK3iej}O9sBbtvWB|na5>aUC zLOd=uuq6#=;Aw3>j1)PDlbUX^aw0~RmJ~yt*IitDp&CplJ|n-rOyobyj-cxIpLl6$ zCCgMBL1yHBAn7AZ@%zh1U~9CN>U+!`A{?Xzg&*2s>9b>SLV7#QS+p3YnmOaxtJmSJ zaU7P+{|1*VQikSF2AlBeH90xq0Hl@t!Ci%gU^TTL-p@OV-o}-v+#AfUuem^ylXLNd zXaQbZxEb#}F$Cq{cW~oEGK`It!LA1xn5(^vdfs(Lr(<96!@oSD?HUb9FUOK-g*|Bh z^&D=myM-mK4?)Sri0Yoy!e?pXj2@WEQrlPZo~@tI?@=snt?-0bt{LE_AB&O>l61ay zCOS1Qqt7~y!NR>i$n)XbaIbEIaMQ(jY>kVDl8v`n<-+^8u=z5Ujh{gb#2=GCMe@}A z)ji>ND;H>Kk7eI7J`jsh({cBp5z%<~mRWce!(r)EoDwTbw)PYsb zwxUvaK>8`JTor)>)?PFzAy#+=;%R8^QHfZH%d1glWPT!qJj9*d+g5&=B_tm9=(~ zKQ~XoxnfOPYTpo1skZZvg0}7eJ_J6FG94gZ=ntOyj^TnBH}o@weAVuw4L- z+WZ9Dcb$N-2h>SfMFi}tvElVSg=E%^TolbyBRhj^fZ9!?V$$Mtdh`R(aT4Y8gEH}M z^=PO{?1HCqx-hnX52{;w;z(ODI2nSRaY>**^E&tC9KO# zi?$34Ld*2!Xl<~6D7S5ZI@Q0#w5}gNRfG-Q8^7bM)?-4d=la81H&0Btl#Xgf=V4Zb zGk9g)!nIyMNt;i(P^-olOVsqZ;ssMact#0#lnvc?5>JM(m$9tRVLUzI>p-_@NTa^V zMs$iQBZK#SY5YAqzM?r5S6YtdaI%yv3<)Df!xnd~8~8o^?=^!J2Qr0y&NldM*m<~J^_PkG6p*9N2ZiDZ>U_J|5Rdvd z4E>9QV6bo|i@CFt&k!Gr*GG%u)ODe-{Q6PQFOvt6dS#SdDoKa!L&&(FgZILq~|9i8<5ObF;3X;fP7QNt2>&h`xOLZw*o)bGAmjc(XI#ZyUSa z!3hfrhu;bE2WlkIV+oR*t0_rJ{)qVe#Tv;k+xhKNw9+T-+A~Svw_?G-s&@X!3}?al z#KK9N+9dIHFP5~Wt+bWX4if};5`s*%5V0!!BmSghA=nYhm)KNr+TSYb3#2CTi$N#8 z!IS3!IHZ zB^TH~1=Fks#Lq?N1>4jpfq}esyYJUd@$7CO5&2GX;b-KEJu0UOYP$RR0qxU8_p;c6 z12dE))TpE6&dg51mL08<*+aRKioSYrjl)jKvKz0(^rwr0hkIjrgNCgV{{B``L|TKW zac_&nGa+$DB4@e;514dM*(4?$N@ zzG%pGiEY^+TT+y6D>3sg<(r28o%D4J1PzK)Y%gy~;72YzAg+3_c5)vz@r!<(kjyzh zAifxrA&|+q$uHNfpVZ-4(zPc+ylr8qV3ENI!SQb8c8mR z#rN3f1haD?-D-t>QJ zwx1_!bvDh@5+pV<+J8izgilCKT3+Bb`DOy(x1+x~8l6Dw#{ z^6gt@wSUU3l049gmpFJ-3dB^L;OV<#l0KTtUlXj;ej#XWd*x)lY@n5G`)ZdA|Alg) zXz*K|_>J5F@%;~zSofm>K2Kdm(%d|mzmKVta7&fiNp6M2tJ+2MdZb&>8sOP(H}i1Z?$~!zNTk3POI} z;>Sy`6^QdDu^2~<_FpO&1R?%elGrqL$)ab9?VpCO3uK7bj2#US-k)5=!^IG>4|Qt z8jH&}@M*bOru-<%NUS1bBsP6#D0VPY6Yo(q^^au0lq;`2eJ^fTKPQKX; z%HmwsgS> zGj5TOuj}%I^b;<4v%(A_m=$ok0fFV{*$IvDm}&7tf+w zk>7MhSzJZui|oel@Dv>_L{Dgc{>WpdND0b{&99a6RDTWf6xYq*Q$D*zvkoVV9)>6J zcdT|2mpgnBok>0*N;{cB$E0i)ou0CmpZwC9&ms=;>)P$bH)6^}r8g#cpH?XFwdN<2 zyT=U0URRCyOBV;B@L%Ed+e2PJEzy%)1^3W#CLam4PdjP#;$-+z-ken1s!Rh7JY;G#{CzRrTC8ODdspDi`_}Z3zVh)Z=`vEn z^OLM1%i$JL{F4k`wT>S+VqncT|Kd%r*1Ca**hZrFj19ae|K!9b9rogVGi`XuwkPS+ zYCgOJ!aC92?j@q&hI^u5jk)5;A|rmmhNHCqlOfTAylmbFaXMZR$`u7{99C+4QoBd^2gyun;LeeHU3my0i1<6KFW z_ZrZt)jee8VJ-=7+rW`y@^Ft{HTG>bByTWO$dX0_+9 z;hB%>OuIQO_53gv@Q2cSmF45te zaiTrfn&9=(yV$A3od4yz3xAfWs>s*Qo>%m7ov0)`g*U%YgTJN7kf;7+gm*AMmMk{A z3Y70;@|JFVPWGCclb?fEinibU!83fo;m(k3p|zqf!N_&#*rLsc{CQ&$yjk^x{Kl(7 zCQB;FyMe~!In6sXBkUhZRL*AVub4q859v>(Xg_uhTJi8Ob zdlVSrfZvz+Z@&fbb57QZ6{G(0V$45_#gUHuRT}pEq1s{b!^S;gJr7#^mHmZBRWSG! zhm<9eqBMS5vl9Q({{4K_Z{|D&xk*mUAGRbkSWVI;vX{8Ln<9AsRENL4sYiU(q*u)A z-zgG-GX8z5$Ksobc`&qXj!5;h5bw*0#rIY3kjUE@l(|jkz58Q;R*wkDgyMMkeZ_py z!7cyD$FhaIhQWK}zu#+M_ij(ZzQ~OpKHo%to4ua5(MK1?w-sTH8`J2Q8?EWbmD#vB zr3cQg*hbGeID=Hm?!fi$T<92fwD5Y77B5mri;}M&V_1$)21l2X&9#x+OwRl~hT&U% z>vEwb;~ikzY#T{o&q)7brK)h6wYTCCYwnek3^U$TmZW<&(?n&JO~sBytg^PhET?D3 zZH5{d)(p94Rv$HLncYQw%%s)&O!s0bmfMxxj888ut&ia-=61u!j6+R9R?4f+F&<}Y zvzC8|wSKHv%iL^X%1oZ$X7kBmiq&uLF&ka)+sxHBv#g&7cQE+}YMI*_nasyYwk)Z# zm(~{oPcd76L8jFyO{Tr79V^*?BWo-#%jQu&wrMH4z!(Z+FzSFxpy?Je_wom>8pyLNCBciPr^PTgtD z&VDz-M9*^Ag+~&&)QKJ3hqn;-d8-Ha??x@o!M*Jq8E4jW*XX<$l*$#p<15>Uuq~(YWNu{GtAk(Y9XN=Cwv0t8@X?>6+xW5yQGedVKG1?v)vGY3;*$YGOaHNtpa-X{J+17idSx+y= zv%ihAIWGM>*qmRZXXb@ua43ffc9QpyO?}l8_9>To#?rHDHYUM~ zSuphhW53ZUW+|`A#)>ipDA?Z<92uO*NBg!^>lCSz*dD z7*=Oq9*$)_Si6K-IAeuP`oD+P!&U~29f^9(zWd)85>tD&WW0^}IRdb|swNiH6`f>R z3@>J?&D(CZu}8wlcRp*i^LH~tv-&AVtf$PBb$Gxom}y5St}SJ1p3>*qr%z!oH(kKt zGtaR)bbfP{2TeE&KQyvy&&6|FaV6_0p}as<4A&#~IU8 z^|^m(>&=9B8kl#@n^+3cPK?!9iCxq_&Q>t0Vs&j)W#4}t%yK$*ih1d> zAM+h=D(6a27CS_t-DXX!A5&LFo%KVunzg@fI(v$2KKuH<6APaOE3=e+OqiZr4(CJW zK`U3~OqSh-sjP(-fsEUB)|@?S7qL&Td}TBBpB4Mdm=iO-lgS~EKDE-3mg6op&|yQ> zLmc+?Ib5)vVtaD0vIpw|7^%to7|Hg=+<6Yc+@`5g?3rJevio<&u%9=)WFMA%;{-eE zvR%DSay~@T%zlP~jSPIpKEO_81>7@Waubr-p27ysvBo7<^x~7u8*mq+cR-pU^(Bqj zk>zdkZq_l@rz=L*W9kMhZi5&2NFInO1NcGHbky7#ZgzNB_8zpaJgE9lZb8mr5IDLX|rQ0_c79AI&9*6_p;c(6Rg~>{9ye_ zH?msTL9i0MhHW0Lw_sn*Xl1!8$a5->9%a!ll5OU(R$J|5oMwho9M*?FlX`lyhs`a0 zIYwGwKVxy=X6CI>4My_jc*gR;i%jX_35G&Y5##5LeaxDJi>)ixOR>i|t2mc>9f*UH z9q{%?HVS&a9ew?LhH6@#PCeas1G#!Wr#!#dVDY+(RIrl)>MREBr9X+A-DkOPv_^@0?%ANd!-k58h(MlhtpJp* z1%}I{Vaw&S-2DOymF`SHomZEE3T-ny?YlgF>M^}7+}8(9`4a#%wa%lk=}vfrT!V|t z?xH2BO!&DfmTPv&9A{-7LyUnj;j#P0#FEt8sP*cV>ebWd>gQjW(TU`xmli= z(D1@{ov+~NYp2MO3IX{wO`4R-aH3yp&c!i7pW)0y-H?&p2XbHj!4`9i$WDe0RqGN9 zls|eCO4aeGw%C~1*LIbo%(4W1Z-*!|BPKFlT}WJH8G$i_wQWVBJn*c)4w=PWCe~KO zaW(ZcQPA~9!rtA1Njy|V--dp06^$Qq4ZE%)t;|L=1GIB1zYS4~KEx2ZH;$l_uga0o zxk_l;6AIQj6ru{R0BW`LVY^iCqey^;u0Th zbm0Es6y#EwN$9QpLiv!(K={QTV!E3lp8MrE$e3MBoL4ZVguCJp%Qzjlir0d%MM4zu z_%rBI^hVP~Gthy69QbAbL+;d_v#?d!6%cuQi|}-A0bUoHj*{ps^iJU2Q5vescgc8*a1{Rs1o^$s(cV_MfhZZ>l#kEmzAWjbXJ>G|_)}DZ? z+CG8}qub!w^;?C9J*E(md8z2htD`t%o-^!x*2s~Xww$t(ZA4w2BJS918`!x(4QF3y zM$AA|BQNIwf+UGNGy)2D>j&M2a; zTN9wj`!^LlDvMuj^@j6TW>e?atOWFzSR!=AQCz%rzVO2F(_l7TN0|&<;l7v(VU*`A zsI%n^*jPJEc@=B~Vc`>COZRbMR*4(-e{BhGna_Yz--g0?^#f>%jXs`#MhUOH@Cf`4 z3dbq|%W$YvD&@RxIl?j&z~)+bXQvC6m?5mk+JsXM-+%!Ink`-*PGA@B*@(GFnHu@d zKxu~~(WewgNQnNz<-|9X&HjMqo)gpCY(uc)#}lN_@};Q#`Y$T+51Y1LWl1kIs3SYt zrAR_5hg|S#CC*8W#bs1HJa90B9{hX|RKGrs--SnE<&7WlIpe!zl*dl8Z=Dt?rL~im z`WT0vrcNQBR(yiRw~|3gI-o6-I!U*l8tz?1Yp8Gfh9>*mz%G2R-ksXGe;i~f&%o=%g~Vdahu@c!qF;xU zvFVK*pl~;n5*=2dp2)`{xabZ_Gn)`j`EnWDj*7;kU$y}EURBJ$X#=7j86ywhyWrc; zG?<}!86CRd&xHjtDC)I5Z1I|f-p{^=7VhweHPI)~bTGM|J>C+oDw)`MwV!Y#?KMy? z3PGu>E8C{@?11-DOd!8l1g?sB$mGo?Q0=J*yA9_Ezh&lOjy?yz*v^C8LN|CIMu_4z z11#SpgSXC*hWqV;@cW-`cs#nEs=qFU8nW(yKcouQi2M&-(Yb=n{M~SY{x_I^NRK-G z%r00BVI_>5Lq1(1Ou&=9ta0G8r z$wUDNy7*XVMN{CU-ePnvVj5m?JOT8EWP{P;G%DIy08FA(F~za~EiV#@_yTt%oS06q zRz@JPu$*d|z72q@KL|Y7fl6iP;-7gf;7`m4LgAYeaH)1gI}UN*`!-VU_oG3{-$bry>n5;t`y-)4YbTg&6Gw2@7omhT#o!Y^1x%}1 z1_r&`ICTl@DE+M4$o&3x6mvr!>@uUNg(1&~FjP)4|B2AX_0iP#nI_2d`x3Ne&vIhK z@)NaZSutvv{X{rtQ3}|ywjT{H&g05`s0NqwTexkHVhAvB(GY4pSYycl8%up;$B7P>!q?#8WC9aqp z1<5-AI=3p36%tmpwQ+Z~CxIEyH=*2{P^jy8gv+%zC&qCdwd|eNBtsGrvq#SnDSau#J4p>? zyDtwp8+~sTBxMrG{Q<E_a;7v8v^By&xFs&6X2|# zNWjrfG<~K$ygT@g%O4*AAB}0sq0x)jD(OWE7x`H9E*JcAv7#7J--s_Rl}O==gc{zz z27J$)hM%(66Q(;ZQ``Mz(W!$I;Bk3Z+pc+^sV&FvQx1z=g@^Wt(Yya9_v0uB{dJh& znlAiGnYNUoNC$atQpYXe6Nb@&r%6b`dKzxH(M&`$vcbEs44{-0hU^|{z|0qNaA@sP zuC(rB;%2W8(%Vu8EKB-;a)K+|eR&j+yEbB%S$b{jv$Wxt@J2%RZnxTQ;~7pp?8J#Og5t#HiSJW3o(SHt0v zYZ0xZjuQLE;o0$6oUp_O&$MeqM7am0@l`-{e>{l{(x!qpcL$+=T@BihRzX#sa|h?H zeF4sbaPI4VA2i;ZhdVw`71ql3pqRwX)ag5`Kw66uN-HYooIa}{j`Q?q)CP-NKy%c;&A)L6H2rD{{`|0_^80g< zS;u~$_0|uotl5h@^DRNW*B7cw{sj2$E`_N#MmVj}4sX3_fKi?ooV%=#TWK>3FL)g_ z+5AP7+OeaQs8ySdQ~p|_huRb5jDJpa_3nPGbkv3nU@4QDnNM)z^c1-D-Vp7v-3)GM zVUWw>m(q5jE9ju#mvG&yTX17=GW^f=5u@=?L<_1y=NTZ!Uo_6Jnhu&ngJAs9n8s|V&2m|h6PXX@tVv*UQF;w_fEL0z8A__dOQ`_tMxr&T2 zFz`?hEnno0u3o!H44jIk_UM+PH$Rq9;pe$%C99RXE|US?op;1WDNm^hzX%kyBZ*+b zM(X)!8+esE54Mr#(c$AyIHlfA)a(|TC=3LVqW#A7xn}|?sChlFAp(=JRCft{;ygSYa?)Z}mPKPpRi(3sS z(LtSTC{)8onmnMw`xE%9!#Fn8`VJon7eI9}mo$46PklJ|E!-U8=pEZT;~gRghGOqyq{FOL!9K-7G>U)+`g=CtA_pi)*NE zVnF!1cQ&_EC`RRa43Cis3Q~k^ATyl_WZz zbcGb#?;u_L6X?JN&h!S8PI9N%fF854;vIh#MDsiE&}%G?kt-{9k-caZuXkbzU1hnF z4m#q(%X3>S8YNooYN2`Ddi}Jty8w3w6=fxBYlKubbR=50MJ1 zl3`wG1HIwdC-P230aY(hp#3byh*cjxfhS}Qa&6hfJ@h^fY}}p*W~jD7)t7Oonp=a5 zmp_9Wp1Q*G7xEx+K7`P&uV~di*@66PYVeiU`7q$RDdm$PjRuoTaRfHDTS%k3jn;4-N;Kqq}S!a6{n* zm_H>4$UpRh6_yUb#dj0lcmEe0ndeOW>WhGSyN{!u>gC|VwN7G znc!Wj7fzE6M*1gjfd}#wj`Xbs))ASg;KmI!@Ax_Bwu>*^o!o)E&w3zWY%d&J5QE=b zw#Un~UZI)!7m0iK)6uRmCbqq3L42#uh8fHN91%id&iouaZ+#+YzPSqi1`A=y6*c^( zq5}SGNQR}-^2GA5d%&%m6X^YOJE$D8114Yo0Vjgvp=*pX?RC(L8xh%CLKWh0VU zld}c5J-Y;sPwzp~r5A|SWgH;)bQ_SYIi-;7?!?{IG)i#D!JpQ@Cpqa)!QuaUah{<* z7Vmlk_1@kBW}9hjYVC!0?7dIsC?2GZjx40_={I1+?nCE3?82MfZ^PC1C*bmhcS+WH zNS8nH!!0+yL5pN-vaE9r2>Yi*ACP^CWGZXhOxhB0{q1@xXUY`F-E|JhUWnr!sJVkw zRgVFepj9xQ(T=`vlf{;HD$vd(75&omg~9H1)L>XSmHTQl$c|cv(s#cG_GLPtdfygw z{j3Jc-fRFJpU1(pKmQ@)&}j7UY8v{ttQ`!_DTctqlW;6o#4CR4!}pQhl;3w#s=&+= zs5pKDbJW&jWuGeUct94SuC4*+&)DM!^(0bGe+#zPUL~@13GDoJ9_k1`Mkzmv19#P} z;6zIa8d@Z3{`7S>xHGJe)?R6&%mPAT4ZH-#o5rXs2drTkvlMOJ{g?P~whJ<5Cli*c z2e9FXY`9~Y8u!yX8c#h@h34y>K-P61z@=anWN>Oep?h{Ku`RP7I+?ve9S6=~XF>!L zyn>dIjb$yoFNFALptZxQHyIRN%z29!}fPD3v@eP5RR$pqE;57G_l}C0&1paKBtXDLi^@ z3s(JWOd9X0N3Dbro@r}FJF0hLCrXu^)8~NHn*8ts%}wO0vQgZV53qBIB{u9X#yJru zaIL=zz4W;OCWci>gFSgj`qxxS7?cCMUM&N3LcWyI2RP7xT_upf=j<=!_WtDRLDUS+~vF;p58l~5{o$S&4Wd#FKw7`UC5vm z>VvR)-Yej4-9$9{lu#>T8j;ccI1qbS5iX)1qKPx|0F*32Dh4w_vh-1yeL0`2xco6v zYVja0RSCJz(kft6U=mVItU(GL@6h^>7pT7%6|uRJ25SFz4bOYA83h;?z=G$u@YHXU zr@SQOVAm@HL_T*rA<3$ONTnOLXlde3y+Dwg*^Z`7n86uGt)ZrMEvkKy3sZ_71EpO` zgtuA@SbvH@VK0^wm3ejOSG*UL@75s7`7M+QLkV8I;t5vdxq_TR0$e-9hHG!-aWiJK zz-qA~d@<&PN{=C^h#G-&Nh}d}sS)+~<$|X^71(DMg@3=u2OJQM^g8bWrew0Efm;q< zd|(^i@F4*+4OhebwH!iq=RUmsd;}$s*-3dE+5vXn+>OI~uY!Q^+wg7de*DBK3DvG+ z(e776xW^wYA^RlrCjZk*EH$V`7ys_W&oVT~2=5Ir?RhlmKCuL!`^ml~D>fV$+73Z)K9F z+++w-w{j{eZVj+m{+NpC4g~6^4xHFk&0x{BN~(849ep+_13Gi2phbIyfOZ-NMMkNp z<8=wuv3(Y}dHN%AF`kYM2FHl8K^{6YF&6-4Ec!i(iFw3~A=#CusrO%c3HgI^;C%X5 z6x?(P&C*y;6W_X+>~3_c1r_Q!T`+HzD3&6xK79gF%lL!R!^Tu(u%*YzT}%vreX?hK{NDu=ptm zqWY;ltMWJvk%~w-s)c_XUIl+@PC@2_`FP8NP$K4f8gL5y!maN5F1!;m177soiZ|zK zV9#n5k@2!sM3oj>Caw?Hz%UZOHqgc30&YuG}S`|z2BMx@(nD&e%EPu{?-=Z+gSA7I$GAq&NuTI3- z4nJVOtc|j)y+oZ_6Uo}zSzdP^I&7J+=n)XUGv*taaSQ`x99PU9IR5ugz zZd!w#Iyux0kqKIoDFsKGC7|WtC}pUA9f+$8&|8IMVExt==2|NdT&2J0f}avh>3&FP zSC_&W@1LWCkuLbuyb^H5Q5SvsiqMv0CsAAdT+H6P07X5pfj1Yu#hOcG!8xn#`10B# z!1>o|)L3T+1Kgj&lF`dKfAr0;uhSK^L7@5=<=l$>mMZG4-FjDF|Zp2`~3luTU;Dcx*osTDMM@8 zCSZSi4f3bQXW`S|G@f^2A6oXti28D_8lAhd5FIN|gVye9ykOmM`svV3ta3MyzQ5ZU zUVC>LR_)1!-&K0(QNN?Oa>^>W&QgmkOZY9^5fca&?KXk(EF-A=xtYG2v4{+dAAwba z$*@a98sY=Wu+DfT^gdgR;O;WoihBZXe(w$fp9W!((HSfnZXW< zAx&J8QA0K+Q(#H)JY4*F0s18(KxStokk>cBPtBvSzmpVfIyXcpyiEnh+D%A(&RN_O zv6!+vD+6c7l%tl8X3ASG1Ft;R09G#5g62t=(4D9n;)Ah0F#UcU6l_t0FH1{W#cOBd zS10S>%$$De$CAa6vXVwull8#md!Nzk2W{Bq;xuUggJ#WJAPeRWYvI+Z%R#2;4ESh$ z5>~0!1UB!}PQ-Y5{Q2q_moAdSpS!*iMdvMruUzu6XY+jAQ~3ZXy@~?0Tlb&< zLJlwMz+85M5C(SG;ccOjc- zdcB+u?bO!LE8L7oi(PD?ihU05Hakq6^_GG?7fz!g<|gj7Ma$8@z2V@(&QN&ZMG<=6 zFzIWs1(13WN`=}6!*umZ>aFM!5!>qLMVs^Q1XeR;IX63b4r2!w!JuKrUjCVr>MeS7c>$29OOD(5f*Fa5`Dq}q|AsF>X+0| zPv_l2ykax7_Tec?x^)&(mHvl_nMvFoUhlXYorjU%{K3|rE28o4YJF5wtO2g3l>vuk zVW>qf92VUOgO@JKqfFlrkW*HGY8B<-xqrK0!zMp?A=(apTg}6EikF1Wh3nwd5L2)+ z$eLK;D2MV43?bj)5wcJC5AH6N#?|s&$fLf7EKJ-6PhKmdAE+{*)AKScXmut#8UxAF z>I}M@Z`)_^*}{NU_f(^6jc4KC%?@PoqKCBq7C(+{nK7aKPeP%ehe4yr8l8@Oz|oq2 z98EhqKon{_6Uo0!k(JVAVs+*wRQ3G_GGWGp1bP;Abp*CU(bTUKN`V*YwZaOqhPRE^Ch)Ay#+wKhv0pJEir!jE}Hhq9u}CZazhtR z=0Etl$ft#-j?m|kF4L5{vAzX;T-QqN{G5f6^$Y4h$Mx`$Oe6+U1IX*y5*&2qeoL(E zX~J~x26!Jv!K!OwWc=A2EEt%KHEl0~XKlZM?Uy9fcO1YB@fW}!cW65o-$g{Zw{mf( zED^Tl9NHvj4aKqroDeN#49G=q8q4IUnD|TJZ2EW0t3RIFUgSY4HY|ytPDYLZ9&HeFcA<#P6iF6+s5TB>l zBL0+Aiqxyab*g5d;8_MFhWzoRTu-bBqVWf_WOQl{35-_`g1xz$;nqFM*h4Rn=qF1l zBSthZmVF3RpRL0GG_C=ekQ7*AtWG^wi$w;k2=wIKe_Ts@89Z%IILz8-3jFAF5Q_psYzeqmEVq+&>|S!a!OL(DLJju2`qEr#6+iP(5V zhz@QMuo@IH{o@+K5|^=n1UNS#bEj9bFLxB3l{H_M{(V&kse)wR?e=Vg2>-!uwg4z zdvybTFhw7Q-L`~%j>oXW)oS$HoWMVPGq?vjwsQ|u+Q2z?cEE~>Ce*K*Lfom}hb&Es zfl>PvptE=%N=T7~5>Gd<fsW(s{^Xy#`UY!Vw8{dO`PhBaZQjnW)G<27nkQ z;py2!RLkEYB=ZtL*svP4Nt^|I`!hhkcQ0jGZ3&66O<+og8d{_6jp{vwM3g|9Ug*f@wrbfV}vjtt&g_FTHFYc}uV;7$6_UmcOGmIbfqc@LQ* zw}JNNg}uGFuEu=mA0x4={AT)zy!O2Y_W4g2N!teTZoe_4-fH_HN1bXS?Ug@rf9}L>iC5y*-;e{YRmZ4c z9RZNru1y3tsGykdm%t9Wk0A8|2fg{?PJEIvr7r!hMw`#{P%C$D26u`MqR}m?)ZC7-}gWP zOCCntw*ZU8(|`kM0vAb#15aZSG|mYJSO2TRQoTx~jNuE=$vFjYZVW~`hxZYCS}x%6 z%Tn~em|d{(zCSuNr5#*&?T-Wh&ckl8ogmTl8crF~N5d3EkN&%l+{%_huC@s^FO5gM z-Rgy<_Wxl-^~>X5n$~#vHH3n<4N=wG!qKg&)wt~TW$y0ly09f=G5)8r7qLCgCFnX=VuU5X`J*MU6(sRQ;*jfXaa_wmZG zcDC!w4A>jt1h{MbiJI#woSbcwF~{B!V6{F%RJXE?R53h6j_8%pyZX1%k<fO^4uQJ7dh3Y(fewh=`poTJmyEz z-l5vOjUC^~-dq>nSgNvU?dS-;_q?36bMGSQ-#@|SfI(Wrq?IhpKFwW6n&WYyB3k`8 z4#c5Qkpd50Sri6RjGUl?vJ-L0s|jhI zP=lqyg+zVYaVQ&5hTf>ZM%_O9&=;ct>Iu<|R9>t?V;}>wU#>&478{{LJi;HRdLUX9 zhBfk-;PX!=Wv`t8rE;2J(gG#C)Jzj>uQ`jJNK7E{rVkYEt4D_xAA^g?W^TRD2!XtE zLC+B-T>R}HS2fU&y38E}1AW&yc5;(goRK-)b2kyZtG+;NxAO;)G7eDpTu9rCT?vRc zYzgBmcM?T&DCjuq0&GVEP!D?$Wer{jUDq42{kt}7ndJ>iPvoFr?oXg;wG>a;myLBc zoW*+o(y{IKQ*eB?7x&y)Htx`$gQT;&F zod=isl)~$U%O~TxBKoCPF5Y^?5_&94hTW$%aL4?UQ0h|*(Ehg*c!g}C_4)dwmh(xd z_Ma8Z`(8w;+NYqfdQIH5GJXW_G$G47OK?*9UgF`lU>Kt*Pd-n( zDhxAgCtU5!LE_(P)E;CG8oW;nZTr80)eh5;#C-u8Z3qR@m95mna%T`3@(}5i@OsCKVCPT~h4#jPCD+cP;%tPPIQgJx znGSAH$w0lES5l!9lQu@x9z2~YOKlkDfq|9*r157C_phcrC?(U-u!jxxJNhGKX7`Zd zO4pzn?pj2VWG^CY^U=@&IZ!xmU=kgWBZdKJ3}){nDb_FZkne{~#u zt=@>f`&|W3`V(PS=l>YG5=SW7I4t*7A&H78M|Ni3_nldGXO;*_hoqEFDoMH&T@q5{ zELY{uog+C)c4wCmNl8MKq>@fP-KFU8`4?v1>$!f9CUa&2=bl^_%C7wLURXB=XC4vJ z^rA76|NiJi-((kH))Z8-3)Gb@w7Zu}-Yb3-^Yo-Gwj0k7s6{JCaGAgEWOAqOvskF)+I|mpo8Mn>T3eM%vsgdpV#pB+UC=#b8m?p4y~q} z*GY>i!f|0qWitC^-C6$jx?QYA;Tnj zC(`Qu#cWaS7JB9Ll~wVAU;N-b`pgln$#lcvV0z@tUD|5!rqG3dh1T#Z=l6b><9F*` zWQQGbCUx0!(RI`!J~Ho(eEwy6hSFNTu-PavU)MF=ts=~9M3&biu7UC&3AK1HRR?%)wXW8{@ zXNt8oz4<>5u*}vBcd>@aDY0+)HnGB)NsJ#ajPc6iGfMt5#kKMmn0k&6^ZnQ&bJc}3Y z6)h4!|8Y$yQ}bzp=Q&H?ak66%RO5_GsR>gfb5fL8KaGC$cOAR<%maE$=S*hNZc~P@ zYQh*a&JjO2lOmix@r`;6*6=?%zviWxbPCNTJeIa%U9oPWt-wtjAy!x2EbiD6)~ftEiB49BXG3-bxBP(BC5pl?s)^Rdti^}`VeOAq754_0XyRG-)lk+&Nhx-A(`jK2wY3wmpvvCXE z5#+|(5UnKqX`ez@F4w0wZ=S=yjB{w?=LI60#xDK_4=sk>7*7i|;`siTA)W63TlAqd zla{&snhyS}&F9rzVb@trXI4$>6J2u17E|Y6(lymQW-wSoe17$Fe#2Tny56Ojt_YW6 z`imNQi8hAffWS^h(@MZb%ywi7FWR&ASGLgar%o1c|IsJ*ufv$1PWOc88Z~L=iS$JdKP54Pi8R%tkDdzVMU84P z7es07y%YuUk+G8M;RacY&#y8CX2rRZq>zbs{gT^)(@FLcaoqunxy2hK`(pn{ss@V$ zSzQH!w45mxLDX)^mfd?K|K$f;6wL{#whPaaytRp!G;bG3j9z9-c9@%4ynm}{VP6tr z(KeAEKR)tNY)`%uY$$v$Ftan_GnQJ|M=D4yuie8UP$*&`pUKs{D42cB3X$py-vYO1U}v6 z?>nQ&MyRUNB$nkSChTccbxpgQCu7uf=9=FX&aZE18R0mxOEB zd2~~)2J`G_9&Pj~j|shPN?(6>P7F7z2~-c6(m&O&GL0f0`$R`&BD!@(Y`;lf68OfG z85v$Cx&+GUtOpy#dq^dg=K8c{Ev-a|B-LgTFxFEUm+foUcndLIL>@KaY}q>p`~~%u}K`ByMoRd z-Or3K-^jd;(i8I(o-mCO70goN1g|XVF&*=Cs@TZfpNX5?${bHu6DX~?%B=O&kQ~%@ z6uP5tj9mFG`po0g{EaqO`I4LFjQ{9$al>J0!SwRQV#Awu;zkQQ3vVA?0rObDT6#ki zbHrF)f<3$;7#Qpl1oxX%*Zql=-0cSwJwVa|vt&$=Po9%_L);>F#WtpBgd@!+7WFB8uY?dM+NpW#e+!K*K{6hGpNei&O0djy!#pt-=9LyzCVXF6`jfPM^YbAALgy)SB=om277x1s@k)ot8^~Jy1*Mshs6~i|-Q2 z9^Oxne7(SaTmDP%XbmN~r7kN`rg{Vd*-j?>U%lk5>Rra6C00VJt`(HqlnS)|%ayo= zLka)%xFkcmkogdBqOJP}Us**TS6ncTzkx)ClMM^W^x^&$N2a1xWM>Bhg?3*fARK zcva>k{o>FP_S&$nNd31WoAvF2sGgg}ZgZ0G0&ySK zzSoWYiLIn7f9i^FEj`CBK8p!QRlo6(^?cwXKNS;+bZM35 zI<{)nTXwTa5P#i~1EQDmwxU_1hv~OZF3>0I4zqSoJNfatox-w|WWL}C*eRLmsSk#=c%4(;w=;+_Vk_)^lS13FZW{`>osI2ni090eHl7Ncjr&z zt^FaSuiE8`dd_f|8Hs)Tk4fLyD8C6$(uqmT#)uMH%Q=m2_EnnIcTg4=6>MkYziZO- zO6Azo3bW{_SXTIKP)A&J$5@1$2Z%CLzp?wyE6~SpycF5H8;JQ~R-$tqO3ab10&#}! zN#3zyW9DMmZT|U=DfE*k-s~c+DSXH5S3;S)dOYm0v$%Mt9PQpbnXxkSVrPuJWt|p@ z*-h@ctnCydTJF|ddi#hc{W!vxO?4RKJ8$eVf9$r2Klj9MS}-ucz7AT*lMS?I6z#i( zlXNGsuS+}l>yj_?+Sw*{lJ6Ae?`jiT`?{t@@Z*h=1}zy$W%^#hx2=yD{&0fiNn;c9 z*!cPc|3e7mpTr1?{#z?qOUX!-9`;JS*0eG8qtb$bNu82RdnpT#jxlCIm9)Ug`5jY{ zV<}kt=Z>VyCQ+gw+aWo#u~d+}=``b!=qs^vUm`JE$cSTC+6w;hFH25ZNAO#0beTOq zl<0yU%}x=oq=QXcultv@2^IEciyF1-*t+eF{Oi|qX@R$jc<6@*8wX?9S1P1v%QIhL z!IpLWxIb6uYI~ z@>Jo6i95ba2KY5zjk{9Hiy{aqsRC@G_5J#czeCBqMG?_=d&1v4cx zgJ|&=M*$mlKs@vI#3WJWdWLsnF@5?AlC1k~z-XTV;$uFe%zq1hFeS|q;?1?K!iJ9X zl97>7+RZYW;Ur2+%5E$WWEx2^2jVOlNkA53)c=T?q9-G1tW6PbRPPYOmQM@`8fdH3 zN8;Dlc8Krj%ZO*!-JxgwwvzaS81duOYUnyvnYDX-O*CtD8NYhpT;atR6Ikh-hVbSR zeYPwpPjt)lB>gi`L9E$b%3gkTO0@7{i#heHU9@JvlP z>B8}gbmr|l{Qu|!^y3`B^I1HHKWoH|9xx4LYp*8oE)+BhJ%?}6ryqFWvL(65sl^4< zwdI4{W*tJbPlouOa~P`IB@mMPHYj+HJCSf*6KPlvqTZ6*;9}!!v}IrgrV!YGJvpg> zj;1xCJtt$~y~IrXwqyf%W56P9yjhb3&A$Jo>X&+>PRnc5vV;nF z_&_{zHy$MsPEz&qnUo)KinOs&;AXFNL)%B%QPAP@z!^ zgEwb$UvzjjLwn&f*tU@0i9Pq`@8Kc*6AIGElxro*3%U1Ig$5OmpPrph*q~jssU>D8vQp`}~VI zR%1i>^olqG=5L9ml42~L4Z~8a_rYi3i{RVA7);acBC$_*95;I{0{u_5;k!dKF#XVL zAn~;#3`xC&fB6suMJb2Cg!ve@M|Q)h!X!MGBZXxP6+m6`OAxwVfKP26#(j25qZ#ww zf%>&?@Ll#<#9Pb{^mN@Nx(}+M-@ArEd%}F;y74qTXi_y$O#2Ls#zVklZ!^ffS_zw7 zB7ywckHj%Bik9Y{z&1%KpiJLG&@qj{?oE~utTELnMjWagP{ks;o>v*iA;$LF!DGH{h#Y`3iZq^dU4aX8*N+S`m9XOTiabBnu!QzZx@Kbph`slxz+lr}Cxi`vC zn^ZpeJYNIp6}Y22APKEksiZ89Penhrs*o3K`_Wy+K=?1Ag^VPUSzHMaKiCWs{e|fM%9})gcqytn@eeK8PymVb zX{g!!5|-R|374k2z@xn_2(9!$-}ZW8hO?vb9NGD>E;A9|d5MP`xaJOiF}wI|5K>NqpQ8bym$&_+ozs zvFlk9m@`EJ+bR?>r|2Wdb}k2HZ2AsE_m!f%xDZU)I~5t1zlPDLL*SCqa-^sej8uAi z@vY1D0G&BBvdoSFBrhht_7OX z7tnpz7i6-oHM%mf|G(eZ$2~Y%nmpQ}!PQx_4Md%YM+c17P;=&bQq0rET)!_;RCj?Q zcj!qzI{N7~x;rC`#HZ4N_7<+=4K%hud$qu-kRkQe+IK*=?Nw1#GyIKP32(wf=xhWWg}dD z>nm6lG9TWXtOmUT8bNIPTr_>6jUa36I)32M7cg5z2AuVif*Q77K&oLD_%wVPmq}QN zzN9&ze<6p!vDa?I+q_gT`gcDhWiBD^`@MKg*bT_$$RIjilQ1l>Lpu-Zz-X!u>XrQk zg!W{De*XxIcM}7L7f~>Cr4XCvy&iT2iqL?@Iq;l)2bIkbcxf~T#DwjG#mD87APCMJU|tb{5s1P=mWg*TAQb2}G>UFv=*d=S*9b1iiW{fZ&T6 zR{1>*>HXG4??SEM);pU3u9^tY)K2(T_c+I4Oqx7m%}4$%cIfNX6KIk9V%U@(V9rj)PDRlVy#J*u)L9g6ZudirE4)W3+-+S<%D*(^_v z94(=iozW%TobHgR!Nb&t9XrU#$;D)-poM$+YCiY-N}9T|^EnxH(U?p+`iMMfD#w-L zy`moMZQ$nBsBwG0{6>PlXv(fMkg}Mfg?JvTsqhO%l;iIR%;`S`RJLO>F%({=P~WUGj!YHW;9>SOEQsWx=nZEx<&54H#02LdTZR#Rq3a!snLP;Xmta zj!f}w;FNHcGdjndSlrSK3Nznx?0+VL9^3QS<`|bddFa73^w{fh+kwWPv4xhQd6-`Tj0+ zWY;3tyj&I;naqMq52T^vr)^-3+zd2j{UsDB--3L>Bop5Bkw%a6l>q+o51KrF0c9Ry z@X-?pzTv###9mxKarSN@F8;`!?=-uOIQIP(Hhc0V&^XB!PczK`vR~9;$@Fz##=p}* z*0BcvbJ7yiT9u1E)_(*t&!&Tl4eL0)_wR8&eBxc9-e_yUM?dBe`eq#{l}ocz!AN_C4%Ahr;V`& zTd?JFS10Zj4XB(Z$Jtrs1CI#pU>3aeVnY_FTPTH4`7`+B zo+l@?GJ?P=)a4L$Ypez00SiJfIep!Q^X&_WVf#^hs;dQ1T!M)(O@GKsmA`IFwaDNTR-h{L8ms#S}fHXTAyj)SrfyObJDw@H5zjqcX_8 zqYjUfdWvlw)FLcDYoJJD0$frn=PtRmj`C;>B@Hv@Q)$<^I$$oWPrH!a-+ zc^~FbVf(gH(nY%|wUL$NzcW50Z}<-SX6DSDH`AV*x4wZ~BuXMP!b9ME{xs5i8IQZQ z<_SeDiQsB<%5pzyZbhlF)2WK)IBGIF4?c8VqQ;dCDeo)(c%Ty(z4#YKxUQLxf0An= zjyQXBvPV^j^`pxP&NTr#@jxE7UQdN*jm)4&zX>RI9)p3NtH6`T3xM<6$wc>sYdAJ& z1;d!980T>z@q6`M+=*6(3oI9*CAVh7mOujKYc~?Ves_UhYY~)~uEpk2IH9XQKotHx zkIh^lg16>rg9z*-kXW1|mbP}{;r<$Mi+w1*T(Sw&KI9Rb&AtE)Z7(#SmrM*n7Q8sr zhO0%G5v%*G;r0urC}X`f1T(If4o@cujcHfFMo$|;Wf0)fxa4$XojZa332%`*CgR*7mtr zx_gI&=C~IGN@KZzcv#7{Hfs3cNWfP}ch# zUT`iJGq#(_(SNrVY8G4qf$r7#L4R$`w!xFw7Zd@u{&s_gYgd7#lWmB*8`csF*G&Ua zchlfe<*A%A(W8WvWET;4Zw~QS90k@keg#E4WRdD{ESeYa0MGZ=2J71sz_g($D86gU^UFRCUk>^?@iNXOyXX~k&MY%g?(8;0?9BS@R@zkMlL4@5u)!0{uf%$DhF(yJVd?5hH+~j%W~)FHB&t6sb(sT zQPe7vBh-A+elvNMKr=u8F3K_MJf&*Nr#i|txuv}_W|9L%+`{5LW{NSB%sw}jldCZw zZt!gMD;y;!XXWTY)PEqoF9bZW?LcLCA-T{z z94&adlsFXe5qznPK%5ijz_Xtr&_sCw`A%XD5%Ue?p56n;9+;A*`^q?~TbyBOe+YQN z*T$bIiqWAlcl3Q{9BdT(6K18KfL{e4k(CU-PvtLq(RL1DMfRxFY7*)8cK}RXbqXc& z`RH5dJ9x%ipWLnU8a)ct!TWa?fIZhLVDPpr$cI;i9xWD;tw-*lo~ON}M$vgrUM7!R z7WfSG5EE_}%kjt?xqdLCN#8JMjfDW^6uQ>pnQ-eg0B6Zwfg1k_U7GbmpnpDfBZrP!#$K)`P!`+Jg6e4Gic^!*-^jzh${{hiRIh=8Kr z6IgEEAsEuq3v9>;^y|)2xEKU@#S2Q>DhsQ zC~O9CHZ($ND$sS=TaJxtImoe50%f%n;T4;Y*1K(letS!Z<_b%6C22ZLzj~Xf9(E(| zB(~s&=GACP=Rd?0PsdJ}E+)gz8o;;CUw|qsg1w*miLD3Iaof^N+?y9miY6t&e;tm)c@zw>cK(t^cco!wJT!*MNCy)qTqa3_)1#nl@5&6;Gy>Uc_h`|DFekRY!(q61+WY20D@zjw*+z;bG4O$YJqHpu}_% z3vnJEl9`7745q*{ua*X!3*@0`1`g6wfOzz zBrZM!nO8S(V>4ImWzpC0%n{1Y+F&Y$e~kqESZ$dM9@ zU^1os2Gmggjgsrm6Gb1B@Bsq@vP8#|EC`TA>to}<_7%(Ea%V%dFBL~Sg3Mu~<8So+ zUl^Qn;4!w^yBZvw=Lp8syz!6ctl`655y+lzi4S%96K|iqgXt6NcpL5mDr8oJ44(x^ zSMr0i;g%oR=-vgVEk6w2$2yT<_X7CvQ5){Czydy9^%Y!kolV+!IFRxcs_<+;B@kRa z1O*aZ3#wo->x=fVq>p;U^rF%TU(OIGX2QU~Vek#kaYsM}5LX#KWZ)UTn_NS?b5 zy_r^zrsm5~8r|~R ztY{!k)y^VugR_K(;TaUGe+&kcpGEe~%IJ=-I%w1T4@f_aMn?+TL7xK`Ji3+-ia$(X zD!)K%NGlHweLa9Ve~cq88#JMvtqq`|=kNrd@E4_AdVp&sR3RU8O)|aB(e#?hYh)zq zAZA=S3|`Ygupu@d->&(RIA@WL89wKe6{iw_XN3s0UU~wzyX-=H3=g4WYu|#QIVE7+ zvKNrjPoZLM64W+~B4Rq#O{J)NM8#8Sv@t`Fv7 z8hezHtztSV+5QFM_J6r2vd>d0QAg9Fdu0gkOM2!6XEFSNwDJYLaY*A zo#1F@fWOD<;NmWOqu1%p(0C{TWqz*$PRc3J*km119oRus>#G3M`y0`*`M=@r{y>xP zb_&n^g26`iI+ipRy8 zg9dHja5Dy$1!$s>2gk9E4hMm%t36Pkk_8*92Z@lZZ@}uADbOk%h7QRLwtDLZ+N%Y_s8->V&U6>*@CM{S)_MJJS>xvAj)tC zv1^Ymvg}~dme?rB{4jtwa&tj+u`|k2fCbM9KzX+hfQ%{HrWrp&VcYh5uzW!yn)*5wee%|XTX+8kw^jyoe;Ij_i+>bT ztdBL!Z2LpLzId73p16@Jaa_y&^*n)^zXMUt!V%w|9BjF9^q=X3Yr z6`XNVhxGKk!S&cQPL=&FqspgCaq)yyXan;}Y?l!=GxR6n&pf5#-!I@=L_EM`W=6t& z`Q?~%xD;wQ)d3vIMaEQ2G58;?8!J;K6PW^zc(C7W1_LMi%Lkj>_kt zkGD2l@Y(_nG;ASfkB4v^V%|cmpGO3QD&T&rKcRlj{ite93aq^v4encwf=opt)YqYd zR~DO)*o$}+_b3s)>UxPjG~~jH?2G7M?{XyjpAD2GV71e5;|-SZ0~ZJONfX?)sgzm3qZ+C` zKb+h}#Zh;)($S5!wWMdxbn^J=MDBpaO$1i7AQ<$WRB*CFmbpFXWt3LwoULFYiTL3oyn}W(O+(LSRjc~t5BH?*? z5Ja_eV8os|q)>ewK33caCAF%s_Toyya@I?9#mWlGddQ#_^Q~~~m;<3#u^vQ5zXyH4 zWs#oTiurabO6ct%PHJB`MO-V}f#f(6;>bQNP&p+728>veCO#iwjQeE#o=Y?F&uJ3c z<9rTmnkS7;&AW%fr&|J%<{)_aZZ8qlH8KADf;<)`xd*!pBzbd3xqFqqBAtDuRPW_Eq;ujE z(x+NTnL3>#cYiA)p9g&7($`zK#jCBUuTC;l*!Jb*z2h>}>73Wpx@~W$XCBYF*#(B& zp1YS&QG6_lF^P(FkWXxD(2M1Aw!m zBepp?#%2BMKrW)-p(Ob ze@uaO7tdqnJHv_W6=AS}I}4nBG>!1Oq6VV9>mc)dfa5ZG2tV@Z0wj0;#Ev!9BgY@g zpy9U~FbtXl-@D|49Y?Pqy$4s&HtPpOYJW2Lcx4job9oGXdh$^7>$s}Feom~r)#UZ=^%MFp#g#H#j8445NP`_TNxhM# zR^4;wo=C{2sxK!{LWP&qqW72JmIftC|Kc06!r(WYHRwpyemur~IbeaU+C&kr*X~w}aAU$EyixlE zgfpE`$}$Cx$MR%!|L`2(ad#u(zn+1+?Uv#P#;qU>wj#=^Wyo{Bal|{#GpK264Z1Kb z9gcnUMbiIi&aW?+Lb#lcK{v0k6DFM#zL;J5Sgl_3wpH z;u2xH_{dtqDSsE9HoXZm9N|JO2MP(@)^aXizE9*YpAWa@E+r1TNr88#4iN@hi(%vR za?YX3pTzsqQm}Yw6j+v%iipRF@GAm?!OCk-O}gQjZLDY zP8pMtdu6G_LMJk1ogt~dDF!`Okl`jq*u$*&N^(!(Q*xm95}8zELK3v63Lzbjx{c*-abWNc!sV9_JyqXfnXrP$>LehhCiOhd_ns~8d zGNo90hRiRn;p9Ct0C{fjiLgIAuxIyWiO$R_Q~MYed{8-xjnQw2mDVO;O+qc9TmBa$ zMZE%-Cvbn@oGQ-8OAm-WiWMLyMhu4!nFI1)HeL+Q5K8sNKtvB2 zc2WqI)3FQ~2h7D+CSJh)Z97LOt?`FB+Mn@RW!WHrxh{snsXZ%r~Xr+hI*i?dBNTc{BjOm=uM^f0rYsnZvoMsYH&&p2J$k zFhCvmg5RI3;NSFcnEdNG9?j*G^cF)Px8e)(gBQ{IcV|(>x_UG;^*z`lzXZ%)69wh# z649o0!|-!eA|BPQOw@Y`LFMdFsNxcdCePjqw`KLCmaCtzou>DJt8pp7RB|wk?14){ z%8^U=5@hMFPkE`{M`Th05vgiMDc(1R`{P~U_y-60CNd8N*hr(I3H=_0pCdD8(PYy5 zMza0-DJZ2;1{tYHGSamidAH7iMI}7AVtWxvSgcBRP3|V2huD#Y{<382lR1ct&IiJ{ zRY>#87RU>H4C*S)$^K7C=#RSr=KM*4EGxDLd-nxGm9_tfvU@G2^L0N0zgHA^9b}HS z_|qJxL;I2K(kDneelzUvMd*+H5az0@0+P?a0rmQ_WOlYTx?=f*Q=CiS8PsOjH|;qF zwr4{3)?(-|_+f%$9VZUUWnodmM3h!`*EG3X2WT$Y3TxITVt&7)P^@1k5!!ncxQ$MR z{hMbKPbXEuQmON}Z^JpX*jx!5wx>{saoB%g} zUx$5|^_93f>j79>fpE83X@twc2*UeY2Oxc3V|BBBgACUp(CFez*v=%;l*6e&-}D{P zHYo}W6)zxOyILW#bTTplDa6^f5wJFK35aQ317AMvLuE(aK-KtKxV+g9d86iDxZO^WF|VjrvS~1jWN5jb_REC#YyUt$vU%J`zom3JY?3tSA()+mCcrf z@8X*I%9*Lj7@5siImVS@BF*514C>C&ddi;JWR|y~-R$C`e9Etlt%nlW4;^GC7VC?b+;?dnW zu=(gHwz5+LyUQIX^xZpfjmkw}+D`+#zi5E)_1FyQH#t!I8wFM@iN{wy)4+qqb>J=g z5aMEFEC~8?84OxI$HKYi2!$_3XzTtk;(b*YsF6#DS;;DR%M)w3n&~CFX2)SW?N#8x zJ1?Qjw{7@0g_ERGeKbsZ{SXF4+5k#c0c>oUjdtHE1gg|Ja`ve*5Ioy}{IRj0B@6aImP6+pEw2%S(1!{oG- z(8!Z5Xj54h2<0(k@~tbx*nD?rUVa-&Y_gzi7?0#V6p$J1M?hg?D)@V|5jMW_M_bY* zC?Lq4Jg3}@q=SRWUoB5e6HgAJ*{K^rv@iy@UGT^BweQ4PH!BBrC>EM2J22ewxU19= zCk5^r*$Y(bh7>CMLIHW=LFom(HWmRl572-m-i z=C0{S+#P>d>~4(?TAJ8JXde6k!n?D3N5Ef_0GhjW zpjEdtavD}AA|-slJy!y%L%TrY^$S3e^~tx0!+#O7iIVp9MIv^eZ~vVhBeMr^Aeq3ZQ>1hq!;z8uFhU2M_9{ z$TJyX;Nz=AG`w#Wc|5lm%yKC~Rv#9CD_4)>mHS&z^j1sq)2pvAMDYtzcWJ^$WNLs! zG7&Is#0GAP?L(*Z8SJoTAyKqe3g#-jhd-jLvBS4zi1{5UaBM>{CJHYFb4LIg9d8D> z`3hp6Y9`l+t^X6)EME+dE;x$jJ{v*p5>v7+@huY8YLVHWFL8ouexQSwmB5h|nfOi1NMiGk zv#6IyfN_DIT)p>p)ZgYI@5h-~Hmj zirW{Fe8yxlZk;NDYiuAxUOXV{Rc>G=#?8dZPzWx$QPAKx0G-K>*oJ#0FdrKvHnr!2 zw!kdl@o*(^XT1yfd{79r^p)VsscnSOvGqhPTn8V@mqT|Z68{&!0LTT}fuM*Jgm{Yv zLYD8L>(fyDGB=B;@cBZ#K5PgZ*4}^{p*!B9?gx{q*CM4+V>~u66Wz|4@Q@h`z^0qS zV3VON;gMMhc^)J|zbxz&m&MyC;~?HUD+Yn1R6PA2Hr zn+|Zwkiw5OJq0ce1S;583-3jBayPW@rSxo5$x6qY+^{Yw%7nhc&3_wC+PXS&ogaRp zZWkWlzR5`Bn(ewu&Gl9_JLfh;mLvlz>SP7i^MH+6S7A4K#m$2H^;L#aPHdpkLwIIG zRyo}FoJZWD?z`MsReI!R-^pCR|7*Jruqcvl3rHA7;t&OtAQGm#tE#&P7>1yzAR+=v zHjM)ef+Ph*L03T}tVl3|n81LTT~sod<07JB&LXD8ydo;%>w#r|)2eU(_xL_&s;lm; zIk!)pdwaTj=Gi-{?RF7*9%n_Lv{2w>A56l>#d4rGnMov6rGWapDR|bblia*Gb*Sxq z6Aw064)|$H@psB=frrK+`1=fH!vz5o4*s>^q&AP?(Kt9gNax>!4j)T~VLs~GR&mhz=XcaN?Y$Q4L z?SWF`S)%BT}cPi z)~-j~^fP45mOOOKsR3V|c@J>UrJ%KGX~gKNhcMH6D7kIuX)pv0fELv@Ft>IVnH0B* zJNs!R956r^Tum0>i*$|2=6gP<(_j^}AD;u}20aB{wTfs&c_}Wt$cxlIlZ`5>kD^h# z3ea<2Gd$C94rLwjLX$*t$TLosTvJkpvd&%vcZzdC=b&|PUm!plhZZ2shE}qqI1f#H z#iC}J`&%}@vn3~0=|Wq@Hq3ed7Ln$d|)X$P@S7 z(T#YVJV%_M2VE_Mn{G`bho{^}3JwP3k$Wm+@JT;%_px;d=B}VSwe{%N{C6aZOCj0u z9w_IC6}f>GLk`@kOj$fnr|(=JLMb2o07d-wBR6tn+AxPdwruqoT4e9gn$&`@! z?msR3w@5`q*pN`>h5@E?L<-zoqYM=yqVXrICNtY*^f;43FJd2i4VwkmpQ> z;TM)4f(o;hpmS^+ciFH!csKJI@!^#^ypU}{c-`HDd#!CCqS9gr7|;l-vuubMBX9Ud zKqGU%VyrQ9BcxK-64ApEC&7x?9G)i;gdsz?HM&_GTIkUN}UPD^AACH z`J;G2at;3Og%4p+>H^b;^syYgDF`2Kc>!-XK8o$nXeRo3KzQKyF2ZG=K6kZoIpLFP z4Vh2=vE;jNCw6uXhD+Nt34cv7YCA?@eGSWrHwWdgGLOs4czl|7q zFc}=;%z?WN)HQ-dD4QZvFgP+{8r>Eeb_l%<~~t+qXaysQvG zE*sKFotQ1AcW!@z`fki52QM=rjUNx7+MnmrhR6EQCWcz{OGk`O(J(|IeMA(NGM;*@ zTnZyM)l;&e3u*ml99%vml{gqXhFEBw4x&!Ff%v)%%czOkplP2raY_(Nblx%t8*8Kq zx3g+Qj_)?`de1VDqppE@=l12A2D}GYMPE>OvILjzFaWtZdW6onUkLe2Z(-mXIoP~S zmuOG{K(?)hsCvPI&H?$*wzLx)KY`0!Z+!(v2v-towyKc7u#2H$%P=&i@jW~?eiw1+ z)=p5`ZVV0QUxf4DFT(g@Eu@;AOj0|Xz}1-PNDveNcDAnq;p&&*xcn?~z$t+K&{%{U z4wM2T+ntfONjzA!@ILH_(jZwD0dPA$7)}o;0;XUwsS)9Y70qx!2hX^Ia_(--;AIp! zb;KC7G_?j^*gX(V*S-kW!$&Z)gNw7L?;w@67NUr94f1`ZIrCnfC#p?pN3m{BXmf}$ zI=!Lmvq4=$ryrRQMXo^)Y}Io~`7>=VQ@?Lm0Jvhz+h#tW5HD>3{>; zwcLwml+d$N(y(>oF&O#5)QVqwk{;Z|rk}l6qjM)7p?oa->2{|;Ds)5%uc>1m)#kp7 zwl3&T-%{#N`@gEh5HL+?>$AMB`fCk{QKkm8+Vt}7V_ zRPYL8A@6#ho#b_`W%Qi2%V<8Q5=D#F(+}^J(lU$2;rR#hNUX{PgEN&;Ez1|m^HuS3 zr+q}-pKO@rz#|9G1+aPdEp%+TK6ycLGvJxB$ODG%M0(8uqHD^2LU3{liW!-Q?!R4* z&(c{B9`5AB3R)dLx}-_U6wg69m?7G^nFCX`i$I3*Xi^8ii)o`&sBmxrG4!`gt}w|G zU2JpUUR*1SKWh#F8cKOs^Q>_Y?0sd?Hu?n=+-}8W4jzZMIHu^gOJ1-az7#Iqp+yKh zw&SfjaWLGw5sp!L3(gg&;Whr*1br%)5RHEfaw5|~;k68KABx)@x_q4LJsTyP@mXTdKOdg8i3#K+DYuqT7Xer>+xK|8kc=q0^M(V7>VeLMNye3wU~#Yb&<~Txr-q6u+s!bD#>i_~&Rm*EE6Hx8fRU z<@uv#7IJ9UwYoHlAxDu(kj z-0?v|U)-my0Nv$;NyhVtV_g#&_p7jPZM`{WB};w zyADsPk+t+z97XI~xC8{ScMy^J`Q&|ZA*@-Hgtoso!T;EA392{0hpZ?aI7`QpT)y1| z45^bMFEw?LDHgO0zfA=-9`cw-lJl;&+_Af@bbQrF58H2EP5t{7SL=3Nq zMyG2%$z7vTaVt#)`18OgcqkGBmT3CS&Sj3Q;58jba$5Z5=T5n0|5_N38^=2~G&<`G_j^S_5g);XpI-m`4 z6Uje5)Q~m0eyC~l{kA4&wWxZ@;%U>itv0%7Peld>TYNAM!21)b5y3$9{ujp_c>fr zYATn`Wj&;3V)-c8DUi;6HJw+ao{PWwScq;!j|SJU;i$c6I21ikv54Nf9-6$8gL@jS z5PQ%hXu+KKTfNx~F0@Od`2h!zl1>znIrSB;60{%sQyYnUcDgW2+8+8n^#|mkTI|vx zB{KilPSmUNPckG0%z^UP@!0w^@k~r(EFTWF=TUKnk}`wILSo ze*q6H`je}`odg_LZ^5Sx-GcWOW#FxZ5YOLM#GKJJ;r@Mp$7e?CfPP2@?8@DNt7^}{ z+S=q{%wtIvEvP}ajqheNvOkh%OIwIz?v|A@)oQ<>u0Ir z%EMM&Tn1dm&BtQWRw4Pl`-y#jE+pot-vni&&*SspVOXTO6t|dFhQ^JoCR)?xp^rQN zMCdT0^xKpvfPQbg$`-r!KOEm&~? zf$iJ0pk2N_Do+>>2dhtk&&wMjyuF)8>QWiXY*2*$2RQy%N=| z(t%tmv*1DvQ)*IBE*xf)NtGU1h)N;j2{fM$crh=6GL1$3>MGST+dj8|G%gc9DUT0!QNWPH7TBcV!4JE)SG6 zJ>e3+0_eB14xf#e;M;g!c(LX-aOz_h*mbHDBt%aq7VpWy#4S?93K0p*-*|#eZi9&X zA0`mdVn?h?l>~m1wqlWEyfNjze6Zkk5%f$ui9OVt1|JzN!Ly#Ff~xEF(9NgavTWoK zsPoi-u&6UfyQK1nyEa>3$~IegdHzwDe9aVAw#yL198VJRUbkW4&^(}K(N1h0*GMeL zl4Y*bxMGBnA{un>7hFqeCvLKEC$xFgN?dqdKww7ts90(^ez3g-TD7^N6E?Nnmw^LO zzH9`MLOmd|*X;r`yzgN7Pb#s=&k(m&Jqv|&o+BDwKY^XIH-N48OX2b42{19y7!2KH z0PY89iW<%}j5ea?|h<%Oh?c@#CEb^$#?z90Qk zD~HY$45t+rsgMIRDk#CcdDM*(0Xi19n7W$dNGq(qj8_S@(P@PQTz``<@IIXcbXH|> zrIiBk)Van4)wTehJ2R2+-);ssIR6Pv4a>o?rE+j&O$Pq%DF^%IZVH|}42Sn>6W~fW zS?v6EdBQutk|=rRg5%k~@Jc`zn1-bSd9N1)`_&A*vsnr0tZ%h++o2BX=on0a^BecZ z{(5AI1rf2{8Q8^_EVNK_B|d>}fd@x?;67FCM^>14gK*DX=o$A9q`A=!v7aWOlW{Dp zY4sjrx=|y_i77(iH%nlUe<;}Uq8(65T)4(!(`kH(c_eh0ugdKsuwt(58h{f` z&2a6hMQCpobDg$J0jez>hCe+ek2#Abph=IHqkdBe*r~r1xQpVz;SWVn?f~?E=J277(NCClH2p_ z;aSX*%2n1udTX8_i~E~N-l&b_(2`eZn1M7}BLAFdIH2xmoSgOiBVb>Sz8QGuk5LAF)R-A#hJ2Ch`>L7m1 zeVrwKIhF8G$^{+zH{gpyGVrbcY~oDW0&L=)RIv4|0^F6cflv!u4ZP=F#V?Om#M4)% zfseoL2W^(E+(MZ|Y_N_m+-1tL73w6`>|Ll>eKa|_I>FIg(4XjErCK;wa9!7}GHZFV zUD;aQD*0u;HCt{e*@03uJAF5KyT(AiZR>lRYBM>ln!NRQt2<7e73i7eSGNh>Y*zFS zt}d<^QWLt{RdD*ij_RXx0&6I{LR%4Ey86~{QPrZ1$g0d!`vvPSFRO8_n9pBpoKbDI zdSunR&gWHuX^X0C+!OhyJ6~5_ym7PgWx*o>s`n``TX(<8H1-`o@!Imr5Rb4*bYXhc zvt7@swmT;B=bQKl`bSKydR;i8x-_=F%IRYz|H^dznnkX+_%|ur>fQQpsvlMB+rr~5 z)ssC3S2sLeU%6$|K*5qw^{NSIu9gSw8gVGh35W)8^W&OK)rQT<$vDEa+E1K<-TKq=zPTH$}&5 z0}P3}cAUN?1iy+ahehQI2O zTg{3&ysA6d8P!vpkl?J!WPSsU39=H5s@A{w#ctJw)m70__BDaNq}@DSOFJX|6M~ne zJNVAyeyv^`(zixiTdMj;aANhoOSN#~Y zr=8~Q-8c$4hb8mE;@{HG%H?@)_dTN98(yH78*Hct`7h|y$}D=;?M(W}iw>$^^KzIY zT13Tny7NZwUqd~5Z$iJ)n$HWA!%5eu3X*CmptP>7KoM(Z(<6%B@bnL0a8AQfN>r49 z8jaVId0`JxnZ^dF7O;UlpqL3a1ni=_!X2SsNglalX)>BIMhE#97$E7O^{CQq6)C+- zhCH9Mk$PCWjJYQ@kV?Ngj69r=p}H7(o{7vuWcb9EqWWy6BXmZfIltx7!kaFXY=kD9 zZZ?XVgdIkQE9-c>TUOZ)I2_K;OE9uYyvVh+o^aJ>KPG2u<}%#IicYp|oTbO_XlC0s z78}^=oJ!%Fb~@PRJy~QkF@1-PLeo-POP?R&3i%cH$*8$)y9_M>8+Yvn%l#OnmLZ&M1z ztJg)8U@ld3K^4t1JjwGbR3qi*IP=CDI`bM^X`Z@UChayk1$B%$P5H{6r~UT6CR<`x z(reUR>9?C2$QOyW)EK7%-tgp^bkzEr^h2u$yzJ@=DCE8cZ8hGR@>gDjSgx1o>+6s5 zw!FDvS9q zE4yPq@yP8Ou}@H~SGi2>Y3_v@kIOS_Ei^aOI@~F?QY8A=G+vM3g^j#uJ$18_RjpzK zFMzk(3VobP7YmYYcBW)oC9iU~$-j2U`T{OvvtwX`bsxNy*SSR5COyv4x<6N!_hhcQ zjh*HYo4NI+R$eW~tRD71Y5n%zZL31-U~5pAVho7EqI+147Jl~xh*Gp(ZKj#+CR z*RVksPCLr)OSD_7er$-8l#!EE_m7d&0ZY!7;%)3g8MZD z1cTP+7_|QhgU&x=(ES`kzt1q}{WJzPOP?dp9NTDdB=f=zII=xfZbUylP=+hT-Yvt{XBlyHq8WO9i-v5LF-N|e;&BX* zO*jfs;`rF`;5fD+YamCzXH|TgrRpqGj&jd{UYq{|?tf19?-KZbvl7ghRb%qys`+}w zbjR143B(|d0u!xZu}~rcX1xu`QaA5yP)x(3w?Q+Zme^VL?vU7Vdm9ovthXVt!+RSN zJIP^6k!HwCaM+&+genVg`U=D2B`jxFbsU?+f*ciLq*xRi9UeV}SqJfAwkiur21SR6 zX9+{XgX0-5B;zG2CQ2L~KO-uR89*@u62y@)!HhUC;-bl-IcmZg(?YseNxXQPa8{(K z8@CQygT>>h#))GSB>w-B4d%u5W7zjKGrMiR(SMC8py$uOz9C{lP(m3`uddW7pcZbAo zNN+=8=hWMf*bVJ%^pdSbB8kI3m&D=!B#F-dEQv0h&$85r&m__HrzKH~HS$}suEKKr zTg;gZ#Sp3g-^!`UNTgbl*-+F5JiJnUM$LWzziKa60Np@5ATl*wqmr9g! z)c5%5{s})vf67mfAMw-kYkvOMDM^X-Z*!0tOYf)osmc(@tCv8C36o^Um``_1;P|ms z{G7&)bQr zs}A(o`t%euzC8tvUr#~f&lEJwkN}Rf6|)KCNLw?TaU5wIW;326&1W_fIMTLkdDcXZ z42inw@GbC)f9FU&^mjcAKgpaw0zZ!9^HoHZKg+Xv^e2u#>-mvyNuVSJ8xsME3Ng?n zl^ugwQrR(JC6yh6@;{Xw6C=q$CODFTOjINTnQ%x3PV61{W%&Ctof?c`cZ|O5{9gMz z`TILv5|7<-Iq7?HDf~n(Cx0rJq94g+(ARSLU#BZ2*1ye9W-QmAmdo$O`pZ5(lZAsn z&BDs8kWVwJB1_DXi;syHMY0uHp&Ys3m>JRW-ACA!v7GX`W!UGI;s3Fm`j;hB8b&as zVQ924Vx}l|O3z8h6prXC>KizPh@xPR41TSJfPKeG5k)%xd&61wO3*@=FrWYd`k zf4@veX5U8il?;Cy8~vj-{?Ai>h3zM&MrQTzr}+xmcc!ua z(KM4d0bilf_zTTvmBhDU*!+9{3GgTU^MB<3TP2YCXRrS|(XsuJ_lX?;ucFiAz4z63 zd#Ap=Fn?IEd?`zkwxmWD_avgw-n6grgK8%V8OrMA7F~7ZaMoA;BO`Q n9`*aS({5q>ky8nz@E%B0KZW$Ej{XZsUHZyCmn0{B`ejzF&X#^D3;+o5-(ZFMRyCCz#LlbGo9xG8%QA)e?1{s97U`4w~EOK8%$PYlA9dB z$i_EiGqVPZ>FF@}E>RgtBcQMr*IY)81|HMApOTj+?`JaQaChBwTU18M1SqF8S&o^X zC6RmWtjUJV1}y*AZn-%*mD!a=Y{iYL$?KSnIg+?Kn?z+KEWpNJVCLrA#H`W4XnNzM z)+$jMVM~~;Tob^y>e|*mpX|qC%X0hJM2^Y*EG8UlUS`bXk`V>DLvHd*7CzCubMKfK UfFPBbfdR}oI6+KmG7oDY0H}ONlK=n! From 7b26d6699a6bd444ce53cfc86493465b0112a4e6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 13 Jul 2023 12:59:44 -0500 Subject: [PATCH 57/77] Adjust location of KerasTensor. --- .../Keras/Engine/KerasTensor.cs | 64 ++++++++ src/TensorFlowNET.Core/Tensors/KerasTensor.cs | 53 ------- .../Tensors/Tensor.Conversions.cs | 17 +- .../Tensors/Tensor.Keras.cs | 27 ++++ src/TensorFlowNET.Core/Tensors/Tensor.cs | 5 - src/TensorFlowNET.Keras/Models/ModelsApi.cs | 25 ++- .../Saving/SavedModel/load.cs | 146 +++++++++--------- 7 files changed, 173 insertions(+), 164 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs delete mode 100644 src/TensorFlowNET.Core/Tensors/KerasTensor.cs create mode 100644 src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs diff --git a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs new file mode 100644 index 00000000..9287284f --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs @@ -0,0 +1,64 @@ +namespace Tensorflow.Keras.Engine; + +/// +/// A representation of a Keras in/output during Functional API construction. +/// +public class KerasTensor +{ + private Tensors _original_tensors; + public Tensors original_tensors + { + get => _original_tensors; + set => _original_tensors = value; + } + + private Shape _inferred_value; + public Shape inferred_value => _inferred_value; + + private string _name; + private TensorSpec _type_spec; + public Shape shape => _type_spec.shape; + public TF_DataType dtype => _type_spec.dtype; + + public KerasTensor(TensorSpec type_spec, Shape inferred_value = null, string name = null) + { + _type_spec = type_spec; + _inferred_value = inferred_value; + _name = name; + } + + public static KerasTensor from_tensor(Tensor tensor) + { + var type_spec = tensor.ToTensorSpec(); + var kt = new KerasTensor(type_spec, name: tensor.name); + kt.original_tensors = tensor; + return kt; + } + + public override string ToString() + => _original_tensors.Length switch + { + > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype}")) + "]", + 1 => $"KerasTensor: shape={_original_tensors.shape} {GetInferredValueString()} dtype={_original_tensors.dtype}", + _ => _original_tensors.ToString(), + }; + + private string GetInferredValueString() + => _inferred_value == null ? "" : ""; + + public static implicit operator Tensors(KerasTensor kt) + => kt._original_tensors; + + public static implicit operator Tensor(KerasTensor kt) + { + Tensor tensor = kt._original_tensors; + tensor.IsFromKerasTensor = true; + return tensor; + } + + public static implicit operator KerasTensor(Tensor tensor) + => from_tensor(tensor); + + public static implicit operator KerasTensor(Tensors tensors) + => from_tensor(tensors.First()); +} diff --git a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs b/src/TensorFlowNET.Core/Tensors/KerasTensor.cs deleted file mode 100644 index 3204b4ac..00000000 --- a/src/TensorFlowNET.Core/Tensors/KerasTensor.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace Tensorflow.Keras.Engine; - -/// -/// A representation of a Keras in/output during Functional API construction. -/// -public class KerasTensor -{ - private Tensors _inferred_value; - public Tensors inferred_value - { - get => _inferred_value; - set => _inferred_value = value; - } - - private string _name; - private TensorSpec _type_spec; - public Shape shape => _type_spec.shape; - public TF_DataType dtype => _type_spec.dtype; - - public KerasTensor(TensorSpec type_spec, string name = null) - { - _type_spec = type_spec; - _name = name; - } - - public static KerasTensor from_tensor(Tensor tensor) - { - var type_spec = tensor.ToTensorSpec(); - var kt = new KerasTensor(type_spec, name: tensor.name); - kt.inferred_value = tensor; - return kt; - } - - public override string ToString() - => _inferred_value.Length switch - { - > 1 => "[" + string.Join(", ", _inferred_value.Select(x => $"")) + "]", - 1 => $"", - _ => _inferred_value.ToString(), - }; - - public static implicit operator Tensors(KerasTensor kt) - => kt._inferred_value; - - public static implicit operator Tensor(KerasTensor kt) - => kt._inferred_value; - - public static implicit operator KerasTensor(Tensor tensor) - => from_tensor(tensor); - - public static implicit operator KerasTensor(Tensors tensors) - => from_tensor(tensors.First()); -} diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs index 18bdc1aa..fdd62aee 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Conversions.cs @@ -14,19 +14,10 @@ limitations under the License. ******************************************************************************/ -using Tensorflow.NumPy; -using System; -using System.Diagnostics.CodeAnalysis; -using System.Text; -using Tensorflow.Framework.Models; -using static Tensorflow.Binding; +namespace Tensorflow; -namespace Tensorflow +public partial class Tensor { - [SuppressMessage("ReSharper", "InvokeAsExtensionMethod")] - public partial class Tensor - { - public TensorSpec ToTensorSpec() - => new TensorSpec(shape, dtype, name); - } + public TensorSpec ToTensorSpec() + => new TensorSpec(shape, dtype, name); } \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs b/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs new file mode 100644 index 00000000..ca946ca4 --- /dev/null +++ b/src/TensorFlowNET.Core/Tensors/Tensor.Keras.cs @@ -0,0 +1,27 @@ +/***************************************************************************** + Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +namespace Tensorflow; + +public partial class Tensor +{ + public bool IsFromKerasTensor { get; set; } + + /// + /// Keras History: (Layer, (node_index, tensor_index)) + /// + public KerasHistory KerasHistory { get; set; } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Tensors/Tensor.cs b/src/TensorFlowNET.Core/Tensors/Tensor.cs index c0e5d435..65e1c857 100644 --- a/src/TensorFlowNET.Core/Tensors/Tensor.cs +++ b/src/TensorFlowNET.Core/Tensors/Tensor.cs @@ -146,11 +146,6 @@ namespace Tensorflow return rank < 0 ? null : shape.dims.Select(x => (int)x).ToArray(); } - /// - /// Keras History: (Layer, (node_index, tensor_index)) - /// - public KerasHistory KerasHistory { get; set; } - /// /// Updates the shape of this tensor. /// diff --git a/src/TensorFlowNET.Keras/Models/ModelsApi.cs b/src/TensorFlowNET.Keras/Models/ModelsApi.cs index 44dca58d..2605c41e 100644 --- a/src/TensorFlowNET.Keras/Models/ModelsApi.cs +++ b/src/TensorFlowNET.Keras/Models/ModelsApi.cs @@ -1,22 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Saving; using Tensorflow.Keras.Saving.SavedModel; -using ThirdParty.Tensorflow.Python.Keras.Protobuf; -namespace Tensorflow.Keras.Models +namespace Tensorflow.Keras.Models; + +public class ModelsApi: IModelsApi { - public class ModelsApi: IModelsApi - { - public Functional from_config(FunctionalConfig config) - => Functional.from_config(config); + public Functional from_config(FunctionalConfig config) + => Functional.from_config(config); - public IModel load_model(string filepath, bool compile = true, LoadOptions? options = null) - { - return KerasLoadModelUtils.load_model(filepath, compile: compile, options: options) as Model; - } + public IModel load_model(string filepath, bool compile = true, LoadOptions? options = null) + { + return KerasLoadModelUtils.load_model(filepath, compile: compile, options: options) as Model; } } diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs index aa763fc2..091dbb81 100644 --- a/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs @@ -1,97 +1,89 @@ -using Google.Protobuf; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Tensorflow.Keras.Engine; +using System.IO; using Tensorflow.Train; using ThirdParty.Tensorflow.Python.Keras.Protobuf; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -namespace Tensorflow.Keras.Saving.SavedModel +namespace Tensorflow.Keras.Saving.SavedModel; + +public class KerasLoadModelUtils { - public class KerasLoadModelUtils + /// + /// Corresponding to keras/saving/save.py/load_model + /// + /// + /// + /// + /// + /// + public static Trackable load_model(string filepath, IDictionary? custom_objects = null, + bool compile = true, LoadOptions? options = null) { - /// - /// Corresponding to keras/saving/save.py/load_model - /// - /// - /// - /// - /// - /// - public static Trackable load_model(string filepath, IDictionary? custom_objects = null, - bool compile = true, LoadOptions? options = null) + using var savingScope = SharedObjectSavingScope.Enter(); + + using var ctx = LoadContext.load_context(options); + + if (!File.Exists(filepath) && !Directory.Exists(filepath)) { - using (SharedObjectSavingScope.Enter()) - { - using (LoadContext.load_context(options)) - { - if (!File.Exists(filepath) && !Directory.Exists(filepath)) - { - throw new IOException($"No file or directory found at {filepath}."); - } - if (Directory.Exists(filepath)) - { - return load(filepath, compile, options); - } - else - { - throw new NotImplementedException("Model load of h5 format has not been supported. Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues if it's needed."); - } - } - } + throw new IOException($"No file or directory found at {filepath}."); } - private static Trackable load(string path, bool compile = true, LoadOptions? options = null) + if (Directory.Exists(filepath)) + { + return load(filepath, compile, options); + } + else { - SavedMetadata metadata = new SavedMetadata(); - var meta_graph_def = Loader.parse_saved_model(path).MetaGraphs[0]; - var object_graph_def = meta_graph_def.ObjectGraphDef; - string path_to_metadata_pb = Path.Combine(path, Constants.SAVED_METADATA_PATH); - if (File.Exists(path_to_metadata_pb)) - { - metadata.MergeFrom(new FileStream(path_to_metadata_pb, FileMode.Open, FileAccess.Read)); - } - else - { - throw new NotImplementedException("SavedModel saved prior to TF 2.5 detected when loading Keras model, please" + - " use higher version or submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues. to let us know you need it."); - } + throw new NotImplementedException("Model load of h5 format has not been supported. Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues if it's needed."); + } + } - if (metadata.Nodes is null || metadata.Nodes.Count == 0) - { - return Loader.load(path, options: options) as Model; - } + private static Trackable load(string path, bool compile = true, LoadOptions? options = null) + { + SavedMetadata metadata; + var meta_graph_def = Loader.parse_saved_model(path).MetaGraphs[0]; + var object_graph_def = meta_graph_def.ObjectGraphDef; + string path_to_metadata_pb = Path.Combine(path, Constants.SAVED_METADATA_PATH); + if (File.Exists(path_to_metadata_pb)) + { + using var stream = new FileStream(path_to_metadata_pb, FileMode.Open, FileAccess.Read); + metadata = SavedMetadata.Parser.ParseFrom(stream); + } + else + { + throw new NotImplementedException("SavedModel saved prior to TF 2.5 detected when loading Keras model, please" + + " use higher version or submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues. to let us know you need it."); + } - var keras_loader = new KerasObjectLoader(metadata, object_graph_def); - keras_loader.load_layers(compile: compile); + if (metadata.Nodes is null || metadata.Nodes.Count == 0) + { + return Loader.load(path, options: options) as Model; + } - Dictionary)> nodes_to_load = new(); - nodes_to_load["root"] = (null, null); - foreach(var item in keras_loader.LoadedNodes) - { - nodes_to_load[keras_loader.get_path(item.Key)] = item.Value; - } - var loaded = Loader.load_partial(path, nodes_to_load, options); + var keras_loader = new KerasObjectLoader(metadata, object_graph_def); + keras_loader.load_layers(compile: compile); - keras_loader.finalize_objects(); - keras_loader.del_tracking(); + Dictionary)> nodes_to_load = new(); + nodes_to_load["root"] = (null, null); + foreach(var item in keras_loader.LoadedNodes) + { + nodes_to_load[keras_loader.get_path(item.Key)] = item.Value; + } + var loaded = Loader.load_partial(path, nodes_to_load, options); - var model = loaded["root"]; + keras_loader.finalize_objects(); + keras_loader.del_tracking(); - if(model is Model && compile) - { - // TODO(Rinne): implement it. - } + var model = loaded["root"]; - if (!tf.Context.executing_eagerly()) - { - // TODO(Rinne): implement it. - } + if (model is Model && compile) + { + // TODO(Rinne): implement it. + } - return model; + if (!tf.Context.executing_eagerly()) + { + // TODO(Rinne): implement it. } + + return model; } } From 03b44c3b502f38509eff6453a0b40c70d114be76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 14 Jul 2023 18:39:58 +0800 Subject: [PATCH 58/77] ignore the LSTMLoad test --- test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs index 299337cd..cb570fc0 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Model/ModelLoadTest.cs @@ -81,6 +81,7 @@ public class ModelLoadTest model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); } + [Ignore] [TestMethod] public void LSTMLoad() { From 3bef87aefcb84379af5e838ed2dcb8cdc897b4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 14 Jul 2023 23:36:12 +0800 Subject: [PATCH 59/77] fix: make the initialization of the layer's name correct --- .../Utils/generic_utils.cs | 14 +++++--- .../InitLayerNameTest.cs | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs diff --git a/src/TensorFlowNET.Keras/Utils/generic_utils.cs b/src/TensorFlowNET.Keras/Utils/generic_utils.cs index 6a59fb88..5402f499 100644 --- a/src/TensorFlowNET.Keras/Utils/generic_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/generic_utils.cs @@ -29,6 +29,7 @@ using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; using Tensorflow.Train; +using System.Text.RegularExpressions; namespace Tensorflow.Keras.Utils { @@ -126,12 +127,15 @@ namespace Tensorflow.Keras.Utils public static string to_snake_case(string name) { - return string.Concat(name.Select((x, i) => + string intermediate = Regex.Replace(name, "(.)([A-Z][a-z0-9]+)", "$1_$2"); + string insecure = Regex.Replace(intermediate, "([a-z])([A-Z])", "$1_$2").ToLower(); + + if (insecure[0] != '_') { - return i > 0 && char.IsUpper(x) && !Char.IsDigit(name[i - 1]) ? - "_" + x.ToString() : - x.ToString(); - })).ToLower(); + return insecure; + } + + return "private" + insecure; } /// diff --git a/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs b/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs new file mode 100644 index 00000000..256eb69c --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/InitLayerNameTest.cs @@ -0,0 +1,33 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.Keras.Layers; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow.Keras.UnitTest +{ + [TestClass] + public class InitLayerNameTest + { + [TestMethod] + public void RNNLayerNameTest() + { + var simpleRnnCell = keras.layers.SimpleRNNCell(1); + Assert.AreEqual("simple_rnn_cell", simpleRnnCell.Name); + var simpleRnn = keras.layers.SimpleRNN(2); + Assert.AreEqual("simple_rnn", simpleRnn.Name); + var lstmCell = keras.layers.LSTMCell(2); + Assert.AreEqual("lstm_cell", lstmCell.Name); + var lstm = keras.layers.LSTM(3); + Assert.AreEqual("lstm", lstm.Name); + } + + [TestMethod] + public void ConvLayerNameTest() + { + var conv2d = keras.layers.Conv2D(8, activation: "linear"); + Assert.AreEqual("conv2d", conv2d.Name); + var conv2dTranspose = keras.layers.Conv2DTranspose(8); + Assert.AreEqual("conv2d_transpose", conv2dTranspose.Name); + } + } +} From 6ec39ba3cbfacb26096903a628db88ece042bf16 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 16 Jul 2023 21:17:40 -0500 Subject: [PATCH 60/77] Fix inferred_value of KerasTensor. #1142 --- src/TensorFlowNET.Core/APIs/tf.reshape.cs | 2 +- src/TensorFlowNET.Core/APIs/tf.tile.cs | 2 +- src/TensorFlowNET.Core/GlobalUsing.cs | 3 +- .../Keras/Engine/KerasTensor.cs | 19 +++++++++--- .../Operations/array_ops.cs | 29 +++++++++++++++++-- src/TensorFlowNET.Core/Tensors/shape_utils.cs | 27 +++++++++++++++++ src/TensorFlowNET.Core/Tensors/tf.constant.cs | 3 ++ src/TensorFlowNET.Core/ops.cs | 11 +++++-- .../Tensorflow.Keras.csproj | 2 +- .../Tensorflow.Binding.UnitTest.csproj | 4 +-- 10 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.reshape.cs b/src/TensorFlowNET.Core/APIs/tf.reshape.cs index 5da7b795..102a8132 100644 --- a/src/TensorFlowNET.Core/APIs/tf.reshape.cs +++ b/src/TensorFlowNET.Core/APIs/tf.reshape.cs @@ -31,6 +31,6 @@ namespace Tensorflow public Tensor reshape(Tensor tensor, object[] shape, string name = null) - => gen_array_ops.reshape(tensor, ops.convert_to_tensor(shape), name); + => array_ops.reshape(tensor, shape, name); } } diff --git a/src/TensorFlowNET.Core/APIs/tf.tile.cs b/src/TensorFlowNET.Core/APIs/tf.tile.cs index 65975ac8..1220230d 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tile.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tile.cs @@ -23,7 +23,7 @@ namespace Tensorflow => gen_array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, object[] multiples, string name = null) - => gen_array_ops.tile(input, ops.convert_to_tensor(multiples), name); + => array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, Shape multiples, string name = null) { diff --git a/src/TensorFlowNET.Core/GlobalUsing.cs b/src/TensorFlowNET.Core/GlobalUsing.cs index 209bc291..7e02c908 100644 --- a/src/TensorFlowNET.Core/GlobalUsing.cs +++ b/src/TensorFlowNET.Core/GlobalUsing.cs @@ -5,4 +5,5 @@ global using System.Collections; global using System.Data; global using System.Linq; global using Tensorflow.Keras.Engine; -global using Tensorflow.Framework.Models; \ No newline at end of file +global using Tensorflow.Framework.Models; +global using static Tensorflow.Binding; \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs index 9287284f..5a264b63 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/KerasTensor.cs @@ -30,21 +30,32 @@ public class KerasTensor public static KerasTensor from_tensor(Tensor tensor) { var type_spec = tensor.ToTensorSpec(); - var kt = new KerasTensor(type_spec, name: tensor.name); + Shape? inferred_value = default; + if (tensor.dtype == TF_DataType.TF_INT32 && tensor.rank < 2) + { + inferred_value = tf.ones(tensor).shape; + } + var kt = new KerasTensor(type_spec, inferred_value: inferred_value, name: tensor.name); kt.original_tensors = tensor; return kt; } + public KerasTensor this[int idx] + => _original_tensors.First()[idx]; + + public KerasTensor this[params Slice[] slices] + => _original_tensors.First()[slices]; + public override string ToString() => _original_tensors.Length switch { - > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype}")) + "]", - 1 => $"KerasTensor: shape={_original_tensors.shape} {GetInferredValueString()} dtype={_original_tensors.dtype}", + > 1 => "[" + string.Join(", ", _original_tensors.Select(x => $"KerasTensor: shape={x.shape} dtype={x.dtype.as_numpy_name()}{GetInferredValueString()}")) + "]", + 1 => $"KerasTensor: shape={_original_tensors.shape} dtype={_original_tensors.dtype.as_numpy_name()}{GetInferredValueString()}", _ => _original_tensors.ToString(), }; private string GetInferredValueString() - => _inferred_value == null ? "" : ""; + => _inferred_value == null ? "" : $" inferred_value={_inferred_value}"; public static implicit operator Tensors(KerasTensor kt) => kt._original_tensors; diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 02bf0e86..9d4647fa 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -137,7 +137,7 @@ namespace Tensorflow if(shape.Length > 1) { shapeTensor = ops.convert_to_tensor(shape, dtypes.int32); - if(shapeTensor.ndim > 1) + if (shapeTensor.ndim > 1) { shapeTensor = array_ops.reshape(shapeTensor, new Shape(-1)); } @@ -304,6 +304,10 @@ namespace Tensorflow { elems_as_tensors.Add(tensor); } + else if (elem is KerasTensor kt) + { + elems_as_tensors.Add(kt); + } else { var elem_tensor = constant_op.constant(elem, dtype: dtype, name: i.ToString()); @@ -404,7 +408,10 @@ namespace Tensorflow => gen_array_ops.reshape(tensor, shape, name: name); public static Tensor reshape(Tensor tensor, object[] shape, string name = null) - => gen_array_ops.reshape(tensor, ops.convert_to_tensor(shape), name: name); + { + var dims = shape_utils.from_object_array(shape); + return gen_array_ops.reshape(tensor, dims, name: name); + } private static Tensor ones_like_impl(T tensor, TF_DataType dtype, string name, bool optimize = true) { @@ -425,6 +432,10 @@ namespace Tensorflow return tf_with(ops.name_scope(name, "ones", new { shape }), scope => { name = scope; + if (shape._shape_tuple().Length == 0) + { + shape = reshape(shape, new Shape(-1)); + } var output = gen_array_ops.fill(shape, constant_op.constant(1.0f, dtype: dtype), name: name); return output; }); @@ -647,6 +658,20 @@ namespace Tensorflow } }); + public static Tensor tile(Tensor input, object[] multiples, string name = null) + { + Shape dims = shape_utils.from_object_array(multiples); + + return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, dims) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + Tmultiples = op.get_attr("Tmultiples") + } + }); + } + public static Tensor zeros_like(Tensor tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true) { return tf_with(ops.name_scope(name, "zeros_like", new Tensor[] { tensor }), scope => diff --git a/src/TensorFlowNET.Core/Tensors/shape_utils.cs b/src/TensorFlowNET.Core/Tensors/shape_utils.cs index 254cdad8..a77dd34c 100644 --- a/src/TensorFlowNET.Core/Tensors/shape_utils.cs +++ b/src/TensorFlowNET.Core/Tensors/shape_utils.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using Tensorflow.Eager; using static Tensorflow.Binding; namespace Tensorflow @@ -13,5 +14,31 @@ namespace Tensorflow throw new NotImplementedException(""); } + + public static Shape from_object_array(object[] shape) + { + var dims = shape.Select(x => + { + if (x is KerasTensor kt && kt.inferred_value != null) + { + return kt.inferred_value.as_int_list()[0]; + } + else if (x is EagerTensor et && et.dtype == TF_DataType.TF_INT32) + { + return et.ToArray()[0]; + } + else if (x is int i) + { + return i; + } + else if (x is long l) + { + return l; + } + throw new NotImplementedException(); + }).ToArray(); + + return new Shape(dims); + } } } diff --git a/src/TensorFlowNET.Core/Tensors/tf.constant.cs b/src/TensorFlowNET.Core/Tensors/tf.constant.cs index 6a62d34a..ac26b3da 100644 --- a/src/TensorFlowNET.Core/Tensors/tf.constant.cs +++ b/src/TensorFlowNET.Core/Tensors/tf.constant.cs @@ -46,6 +46,9 @@ namespace Tensorflow public Tensor ones(Shape shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) => array_ops.ones(shape, dtype, name); + public Tensor ones(Tensor shape, TF_DataType dtype = TF_DataType.TF_FLOAT, string name = null) + => array_ops.ones(shape, dtype, name); + public Tensor size(Tensor input, string name = null, TF_DataType out_type = TF_DataType.TF_INT32) => array_ops.size(input, diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index c624c990..351fd18f 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -144,11 +144,18 @@ namespace Tensorflow } if (!graph.building_function) { - throw new RuntimeError("Attempting to capture an EagerTensor without building a function."); - // return eager_tensor.AsPlaceholder(name: name); + // throw new RuntimeError("Attempting to capture an EagerTensor without building a function."); + return eager_tensor.AsPlaceholder(name: name); } } } + else if (value is KerasTensor kt) + { + if (kt.inferred_value != null) + { + return convert_to_tensor(kt.inferred_value, dtype: kt.dtype, name: name); + } + } // graph mode Tensor ret = value switch diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index c7fa7711..eeb7c559 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -141,7 +141,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index 240960c9..7a6a7f92 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -41,8 +41,8 @@ - - + + From 03472997e43ab36d447ca520907ee8dffcc03edc Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Jul 2023 07:01:51 -0500 Subject: [PATCH 61/77] Fix tf.reverse. --- src/TensorFlowNET.Core/APIs/tf.array.cs | 15 +++++++++------ src/TensorFlowNET.Core/Operations/array_ops.cs | 18 +++++++++++++----- .../ManagedAPI/ArrayOpsTest.cs | 13 +++++++++++++ 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.array.cs b/src/TensorFlowNET.Core/APIs/tf.array.cs index ecac37eb..4d9c3da5 100644 --- a/src/TensorFlowNET.Core/APIs/tf.array.cs +++ b/src/TensorFlowNET.Core/APIs/tf.array.cs @@ -162,14 +162,17 @@ namespace Tensorflow /// Reverses specific dimensions of a tensor. /// /// - /// + /// The indices of the dimensions to reverse. Must be in the range [-rank(tensor), rank(tensor)). /// /// - public Tensor reverse(Tensor tensor, int[] axis, string name = null) - => gen_array_ops.reverse(tensor, ops.convert_to_tensor(axis), name: name); - - public Tensor reverse(Tensor tensor, Tensor axis, string name = null) - => gen_array_ops.reverse(tensor, axis, name: name); + public Tensor reverse(Tensor tensor, Axis axis, string name = null) + { + if (axis.IsScalar) + { + axis = new Axis(axis.axis); + } + return array_ops.reverse(tensor, axis, name: name); + } /// /// Returns the rank of a tensor. diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 9d4647fa..f80dcd2c 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -413,6 +413,16 @@ namespace Tensorflow return gen_array_ops.reshape(tensor, dims, name: name); } + public static Tensor reverse(Tensor tensor, Tensor axis, string name = null) + => tf.Context.ExecuteOp("ReverseV2", name, new ExecuteOpArgs(tensor, axis) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + Tidx = op.get_attr("Tidx") + } + }); + private static Tensor ones_like_impl(T tensor, TF_DataType dtype, string name, bool optimize = true) { return tf_with(ops.name_scope(name, "ones_like", new { tensor }), scope => @@ -658,11 +668,9 @@ namespace Tensorflow } }); - public static Tensor tile(Tensor input, object[] multiples, string name = null) + /*public static Tensor tile(Tensor input, Shape multiples, string name = null) { - Shape dims = shape_utils.from_object_array(multiples); - - return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, dims) + return tf.Context.ExecuteOp("Tile", name, new ExecuteOpArgs(input, multiples) { GetGradientAttrs = (op) => new { @@ -670,7 +678,7 @@ namespace Tensorflow Tmultiples = op.get_attr("Tmultiples") } }); - } + }*/ public static Tensor zeros_like(Tensor tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true) { diff --git a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs index 72f598e4..675689bb 100644 --- a/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs +++ b/test/TensorFlowNET.UnitTest/ManagedAPI/ArrayOpsTest.cs @@ -2,6 +2,7 @@ using Tensorflow.NumPy; using Tensorflow; using static Tensorflow.Binding; +using System.Linq; namespace TensorFlowNET.UnitTest.ManagedAPI { @@ -92,5 +93,17 @@ namespace TensorFlowNET.UnitTest.ManagedAPI Assert.AreEqual(ta.read(1).numpy(), 20f); Assert.AreEqual(ta.read(2).numpy(), 30f); } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/reverse + /// + [TestMethod] + public void ReverseArray() + { + var a = tf.random.normal((2, 3)); + var b = tf.reverse(a, -1); + Assert.IsTrue(Equal(a[0].ToArray().Reverse().ToArray(), b[0].ToArray())); + Assert.IsTrue(Equal(a[1].ToArray().Reverse().ToArray(), b[1].ToArray())); + } } } From fa5d19dcdab55d7b81afc614f9929bc85c52cb20 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 18 Jul 2023 07:08:39 -0500 Subject: [PATCH 62/77] fix unit test. --- src/TensorFlowNET.Core/APIs/tf.tile.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.tile.cs b/src/TensorFlowNET.Core/APIs/tf.tile.cs index 1220230d..a3b497e8 100644 --- a/src/TensorFlowNET.Core/APIs/tf.tile.cs +++ b/src/TensorFlowNET.Core/APIs/tf.tile.cs @@ -23,7 +23,7 @@ namespace Tensorflow => gen_array_ops.tile(input, multiples, name); public Tensor tile(Tensor input, object[] multiples, string name = null) - => array_ops.tile(input, multiples, name); + => array_ops.tile(input, constant_op.constant(shape_utils.from_object_array(multiples).dims), name); public Tensor tile(Tensor input, Shape multiples, string name = null) { From 0c9437afcb9cc5852abcbd31bcb85c08afef0ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 18 Jul 2023 23:31:45 +0800 Subject: [PATCH 63/77] feat: add Bidirectional layer --- .../ArgsDefinition/Rnn/BidirectionalArgs.cs | 20 ++ .../Keras/ArgsDefinition/Rnn/LSTMArgs.cs | 5 + .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 5 + .../Keras/ArgsDefinition/Rnn/WrapperArgs.cs | 24 ++ .../Keras/Layers/ILayersApi.cs | 14 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 14 + .../Layers/Rnn/BaseWrapper.cs | 33 +++ .../Layers/Rnn/Bidirectional.cs | 276 ++++++++++++++++++ src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs | 31 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 11 +- .../Layers/Rnn.Test.cs | 13 +- 11 files changed, 428 insertions(+), 18 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs new file mode 100644 index 00000000..d658a82e --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/BidirectionalArgs.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class BidirectionalArgs : AutoSerializeLayerArgs + { + [JsonProperty("layer")] + public ILayer Layer { get; set; } + [JsonProperty("merge_mode")] + public string? MergeMode { get; set; } + [JsonProperty("backward_layer")] + public ILayer BackwardLayer { get; set; } + public NDArray Weights { get; set; } + } + +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index d816b0ff..a6beb77e 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -5,5 +5,10 @@ // TODO: maybe change the `RNNArgs` and implement this class. public bool UnitForgetBias { get; set; } public int Implementation { get; set; } + + public LSTMArgs Clone() + { + return (LSTMArgs)MemberwiseClone(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index b84d30d3..d0b73ba4 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -40,5 +40,10 @@ namespace Tensorflow.Keras.ArgsDefinition public bool ZeroOutputForMask { get; set; } = false; [JsonProperty("recurrent_dropout")] public float RecurrentDropout { get; set; } = .0f; + + public RNNArgs Clone() + { + return (RNNArgs)MemberwiseClone(); + } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs new file mode 100644 index 00000000..ec8e16d5 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/WrapperArgs.cs @@ -0,0 +1,24 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; + + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class WrapperArgs : AutoSerializeLayerArgs + { + [JsonProperty("layer")] + public ILayer Layer { get; set; } + + public WrapperArgs(ILayer layer) + { + Layer = layer; + } + + public static implicit operator WrapperArgs(BidirectionalArgs args) + => new WrapperArgs(args.Layer); + } + +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 1670f9d1..b8aff5fb 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -258,7 +258,19 @@ namespace Tensorflow.Keras.Layers float dropout = 0f, float recurrent_dropout = 0f, bool reset_after = true); - + + /// + /// Bidirectional wrapper for RNNs. + /// + /// `keras.layers.RNN` instance, such as `keras.layers.LSTM` or `keras.layers.GRU` + /// automatically. + /// + public ILayer Bidirectional( + ILayer layer, + string merge_mode = "concat", + NDArray weights = null, + ILayer backward_layer = null); + public ILayer Subtract(); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index cb85bbba..a04a9c05 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -908,6 +908,20 @@ namespace Tensorflow.Keras.Layers ResetAfter = reset_after }); + public ILayer Bidirectional( + ILayer layer, + string merge_mode = "concat", + NDArray weights = null, + ILayer backward_layer = null) + => new Bidirectional(new BidirectionalArgs + { + Layer = layer, + MergeMode = merge_mode, + Weights = weights, + BackwardLayer = backward_layer + }); + + /// /// /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs b/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs new file mode 100644 index 00000000..737f88cd --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/BaseWrapper.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Abstract wrapper base class. Wrappers take another layer and augment it in various ways. + /// Do not use this class as a layer, it is only an abstract base class. + /// Two usable wrappers are the `TimeDistributed` and `Bidirectional` wrappers. + /// + public abstract class Wrapper: Layer + { + public ILayer _layer; + public Wrapper(WrapperArgs args):base(args) + { + _layer = args.Layer; + } + + public virtual void Build(KerasShapesWrapper input_shape) + { + if (!_layer.Built) + { + _layer.build(input_shape); + } + built = true; + } + + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs new file mode 100644 index 00000000..6114d9c7 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Layers +{ + /// + /// Bidirectional wrapper for RNNs. + /// + public class Bidirectional: Wrapper + { + BidirectionalArgs _args; + RNN _forward_layer; + RNN _backward_layer; + RNN _layer; + bool _support_masking = true; + int _num_constants = 0; + bool _return_state; + bool _stateful; + bool _return_sequences; + InputSpec _input_spec; + RNNArgs _layer_args_copy; + public Bidirectional(BidirectionalArgs args):base(args) + { + _args = args; + if (_args.Layer is not ILayer) + throw new ValueError( + "Please initialize `Bidirectional` layer with a " + + $"`tf.keras.layers.Layer` instance. Received: {_args.Layer}"); + + if (_args.BackwardLayer is not null && _args.BackwardLayer is not ILayer) + throw new ValueError( + "`backward_layer` need to be a `tf.keras.layers.Layer` " + + $"instance. Received: {_args.BackwardLayer}"); + if (!new List { "sum", "mul", "ave", "concat", null }.Contains(_args.MergeMode)) + { + throw new ValueError( + $"Invalid merge mode. Received: {_args.MergeMode}. " + + "Merge mode should be one of " + + "{\"sum\", \"mul\", \"ave\", \"concat\", null}" + ); + } + if (_args.Layer is RNN) + { + _layer = _args.Layer as RNN; + } + else + { + throw new ValueError( + "Bidirectional only support RNN instance such as LSTM or GRU"); + } + _return_state = _layer.Args.ReturnState; + _return_sequences = _layer.Args.ReturnSequences; + _stateful = _layer.Args.Stateful; + _layer_args_copy = _layer.Args.Clone(); + // We don't want to track `layer` since we're already tracking the two + // copies of it we actually run. + // TODO(Wanglongzhi2001), since the feature of setattr_tracking has not been implemented. + // _setattr_tracking = false; + // super().__init__(layer, **kwargs) + // _setattr_tracking = true; + + // Recreate the forward layer from the original layer config, so that it + // will not carry over any state from the layer. + var actualType = _layer.GetType(); + if (actualType == typeof(LSTM)) + { + var arg = _layer_args_copy as LSTMArgs; + _forward_layer = new LSTM(arg); + } + // TODO(Wanglongzhi2001), add GRU if case. + else + { + _forward_layer = new RNN(_layer.Cell, _layer_args_copy); + } + //_forward_layer = _recreate_layer_from_config(_layer); + if (_args.BackwardLayer is null) + { + _backward_layer = _recreate_layer_from_config(_layer, go_backwards:true); + } + else + { + _backward_layer = _args.BackwardLayer as RNN; + } + _forward_layer.Name = "forward_" + _forward_layer.Name; + _backward_layer.Name = "backward_" + _backward_layer.Name; + _verify_layer_config(); + + void force_zero_output_for_mask(RNN layer) + { + layer.Args.ZeroOutputForMask = layer.Args.ReturnSequences; + } + + force_zero_output_for_mask(_forward_layer); + force_zero_output_for_mask(_backward_layer); + + if (_args.Weights is not null) + { + var nw = len(_args.Weights); + _forward_layer.set_weights(_args.Weights[$":,{nw / 2}"]); + _backward_layer.set_weights(_args.Weights[$"{nw / 2},:"]); + } + + _input_spec = _layer.InputSpec; + } + + private void _verify_layer_config() + { + if (_forward_layer.Args.GoBackwards == _backward_layer.Args.GoBackwards) + { + throw new ValueError( + "Forward layer and backward layer should have different " + + "`go_backwards` value." + + "forward_layer.go_backwards = " + + $"{_forward_layer.Args.GoBackwards}," + + "backward_layer.go_backwards = " + + $"{_backward_layer.Args.GoBackwards}"); + } + if (_forward_layer.Args.Stateful != _backward_layer.Args.Stateful) + { + throw new ValueError( + "Forward layer and backward layer are expected to have "+ + $"the same value for attribute stateful, got "+ + $"{_forward_layer.Args.Stateful} for forward layer and "+ + $"{_backward_layer.Args.Stateful} for backward layer"); + } + if (_forward_layer.Args.ReturnState != _backward_layer.Args.ReturnState) + { + throw new ValueError( + "Forward layer and backward layer are expected to have " + + $"the same value for attribute return_state, got " + + $"{_forward_layer.Args.ReturnState} for forward layer and " + + $"{_backward_layer.Args.ReturnState} for backward layer"); + } + if (_forward_layer.Args.ReturnSequences != _backward_layer.Args.ReturnSequences) + { + throw new ValueError( + "Forward layer and backward layer are expected to have " + + $"the same value for attribute return_sequences, got " + + $"{_forward_layer.Args.ReturnSequences} for forward layer and " + + $"{_backward_layer.Args.ReturnSequences} for backward layer"); + } + } + + private RNN _recreate_layer_from_config(RNN layer, bool go_backwards = false) + { + var config = layer.get_config() as RNNArgs; + var cell = layer.Cell; + if (go_backwards) + { + config.GoBackwards = !config.GoBackwards; + } + var actualType = layer.GetType(); + if (actualType == typeof(LSTM)) + { + var arg = config as LSTMArgs; + return new LSTM(arg); + } + else + { + return new RNN(cell, config); + } + } + + public override void build(KerasShapesWrapper input_shape) + { + _buildInputShape = input_shape; + tf_with(ops.name_scope(_forward_layer.Name), scope=> + { + _forward_layer.build(input_shape); + }); + tf_with(ops.name_scope(_backward_layer.Name), scope => + { + _backward_layer.build(input_shape); + }); + built = true; + } + + protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + // `Bidirectional.call` implements the same API as the wrapped `RNN`. + + Tensors forward_inputs; + Tensors backward_inputs; + Tensors forward_state; + Tensors backward_state; + // if isinstance(inputs, list) and len(inputs) > 1: + if (inputs.Length > 1) + { + // initial_states are keras tensors, which means they are passed + // in together with inputs as list. The initial_states need to be + // split into forward and backward section, and be feed to layers + // accordingly. + forward_inputs = new Tensors { inputs[0] }; + backward_inputs = new Tensors { inputs[0] }; + var pivot = (len(inputs) - _num_constants) / 2 + 1; + // add forward initial state + forward_inputs.Concat(new Tensors { inputs[$"1:{pivot}"] }); + if (_num_constants != 0) + // add backward initial state + backward_inputs.Concat(new Tensors { inputs[$"{pivot}:"] }); + else + { + // add backward initial state + backward_inputs.Concat(new Tensors { inputs[$"{pivot}:{-_num_constants}"] }); + // add constants for forward and backward layers + forward_inputs.Concat(new Tensors { inputs[$"{-_num_constants}:"] }); + backward_inputs.Concat(new Tensors { inputs[$"{-_num_constants}:"] }); + } + forward_state = null; + backward_state = null; + } + else if (state is not null) + { + // initial_states are not keras tensors, eg eager tensor from np + // array. They are only passed in from kwarg initial_state, and + // should be passed to forward/backward layer via kwarg + // initial_state as well. + forward_inputs = inputs; + backward_inputs = inputs; + var half = len(state) / 2; + forward_state = state[$":{half}"]; + backward_state = state[$"{half}:"]; + } + else + { + forward_inputs = inputs; + backward_inputs = inputs; + forward_state = null; + backward_state = null; + } + var y = _forward_layer.Apply(forward_inputs, forward_state); + var y_rev = _backward_layer.Apply(backward_inputs, backward_state); + + Tensors states = new(); + if (_return_state) + { + states = y["1:"] + y_rev["1:"]; + y = y[0]; + y_rev = y_rev[0]; + } + + if (_return_sequences) + { + int time_dim = _forward_layer.Args.TimeMajor ? 0 : 1; + y_rev = keras.backend.reverse(y_rev, time_dim); + } + Tensors output; + if (_args.MergeMode == "concat") + output = keras.backend.concatenate(new Tensors { y.Single(), y_rev.Single() }); + else if (_args.MergeMode == "sum") + output = y.Single() + y_rev.Single(); + else if (_args.MergeMode == "ave") + output = (y.Single() + y_rev.Single()) / 2; + else if (_args.MergeMode == "mul") + output = y.Single() * y_rev.Single(); + else if (_args.MergeMode is null) + output = new Tensors { y.Single(), y_rev.Single() }; + else + throw new ValueError( + "Unrecognized value for `merge_mode`. " + + $"Received: {_args.MergeMode}" + + "Expected values are [\"concat\", \"sum\", \"ave\", \"mul\"]"); + if (_return_state) + { + if (_args.MergeMode is not null) + return new Tensors { output.Single(), states.Single()}; + } + return output; + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index b5d58324..c766e8d6 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -3,6 +3,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Common.Types; using Tensorflow.Common.Extensions; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Layers { @@ -14,15 +15,15 @@ namespace Tensorflow.Keras.Layers /// public class LSTM : RNN { - LSTMArgs args; + LSTMArgs _args; InputSpec[] _state_spec; InputSpec _input_spec; bool _could_use_gpu_kernel; - + public LSTMArgs Args { get => _args; } public LSTM(LSTMArgs args) : base(CreateCell(args), args) { - this.args = args; + _args = args; _input_spec = new InputSpec(ndim: 3); _state_spec = new[] { args.Units, args.Units }.Select(dim => new InputSpec(shape: (-1, dim))).ToArray(); _could_use_gpu_kernel = args.Activation == keras.activations.Tanh @@ -71,7 +72,7 @@ namespace Tensorflow.Keras.Layers var single_input = inputs.Single; var input_shape = single_input.shape; - var timesteps = args.TimeMajor ? input_shape[0] : input_shape[1]; + var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; _maybe_reset_cell_dropout_mask(Cell); @@ -87,26 +88,26 @@ namespace Tensorflow.Keras.Layers inputs, initial_state, constants: null, - go_backwards: args.GoBackwards, + go_backwards: _args.GoBackwards, mask: mask, - unroll: args.Unroll, + unroll: _args.Unroll, input_length: ops.convert_to_tensor(timesteps), - time_major: args.TimeMajor, - zero_output_for_mask: args.ZeroOutputForMask, - return_all_outputs: args.ReturnSequences + time_major: _args.TimeMajor, + zero_output_for_mask: _args.ZeroOutputForMask, + return_all_outputs: _args.ReturnSequences ); Tensor output; - if (args.ReturnSequences) + if (_args.ReturnSequences) { - output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, args.GoBackwards); + output = keras.backend.maybe_convert_to_ragged(false, outputs, (int)timesteps, _args.GoBackwards); } else { output = last_output; } - if (args.ReturnState) + if (_args.ReturnState) { return new Tensor[] { output }.Concat(states).ToArray().ToTensors(); } @@ -115,5 +116,11 @@ namespace Tensorflow.Keras.Layers return output; } } + + public override IKerasConfig get_config() + { + return _args; + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 0e81d20e..c1922261 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -31,7 +31,9 @@ namespace Tensorflow.Keras.Layers protected IVariableV1 _kernel; protected IVariableV1 _bias; private IRnnCell _cell; - protected IRnnCell Cell + + public RNNArgs Args { get => _args; } + public IRnnCell Cell { get { @@ -570,10 +572,13 @@ namespace Tensorflow.Keras.Layers var input_shape = array_ops.shape(inputs); var batch_size = _args.TimeMajor ? input_shape[1] : input_shape[0]; var dtype = input.dtype; - Tensors init_state = Cell.GetInitialState(null, batch_size, dtype); - return init_state; } + + public override IKerasConfig get_config() + { + return _args; + } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 5f7bd574..03159346 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Tensorflow.Common.Types; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; @@ -38,8 +39,6 @@ namespace Tensorflow.Keras.UnitTest.Layers var cells = new IRnnCell[] { tf.keras.layers.SimpleRNNCell(4), tf.keras.layers.SimpleRNNCell(5) }; var stackedRNNCell = tf.keras.layers.StackedRNNCells(cells); var (output, state) = stackedRNNCell.Apply(inputs, states); - Console.WriteLine(output); - Console.WriteLine(state.shape); Assert.AreEqual((32, 5), output.shape); Assert.AreEqual((32, 4), state[0].shape); } @@ -108,6 +107,7 @@ namespace Tensorflow.Keras.UnitTest.Layers var inputs = tf.random.normal((32, 10, 8)); var cell = tf.keras.layers.SimpleRNNCell(10, dropout: 0.5f, recurrent_dropout: 0.5f); var rnn = tf.keras.layers.RNN(cell: cell); + var cgf = rnn.get_config(); var output = rnn.Apply(inputs); Assert.AreEqual((32, 10), output.shape); @@ -145,5 +145,14 @@ namespace Tensorflow.Keras.UnitTest.Layers Assert.AreEqual((32, 4), output.shape); } + + [TestMethod] + public void Bidirectional() + { + var bi = tf.keras.layers.Bidirectional(keras.layers.LSTM(10, return_sequences:true)); + var inputs = tf.random.normal((32, 10, 8)); + var outputs = bi.Apply(inputs); + Assert.AreEqual((32, 10, 20), outputs.shape); + } } } From 737910df9e3eca18e094a2bffefa5516efc9ebf3 Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:23:08 +0800 Subject: [PATCH 64/77] Fix: model.load_weights --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index 8ac9fddf..dd6609bc 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -133,10 +133,8 @@ namespace Tensorflow.Keras.Saving long g = H5G.open(f, name); var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); foreach (var i_ in weight_names) - { - var vm = Regex.Replace(i_, "/", "$"); - vm = i_.Split('/')[0] + "/$" + vm.Substring(i_.Split('/')[0].Length + 1, i_.Length - i_.Split('/')[0].Length - 1); - (success, Array result) = Hdf5.ReadDataset(g, vm); + { + (success, Array result) = Hdf5.ReadDataset(g, i_); if (success) weight_values.Add(np.array(result)); } @@ -196,9 +194,14 @@ namespace Tensorflow.Keras.Saving var tensor = val.AsTensor(); if (name.IndexOf("/") > 1) { - var crDataGroup = Hdf5.CreateOrOpenGroup(g, Hdf5Utils.NormalizedName(name.Split('/')[0])); - var _name = Regex.Replace(name.Substring(name.Split('/')[0].Length, name.Length - name.Split('/')[0].Length), "/", "$"); - WriteDataset(crDataGroup, _name, tensor); + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length; i++) + { + if (i == name_split.Length - 1) break; + crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); Hdf5.CloseGroup(crDataGroup); } else From 05dbe134f8f00fa62aa9cda2337891f4ce66c453 Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:32:33 +0800 Subject: [PATCH 65/77] Update hdf5_format.cs --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 707 +++++++++--------- 1 file changed, 353 insertions(+), 354 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index dd6609bc..c80f653f 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -1,355 +1,354 @@ -using System; -using System.Collections.Generic; -using System.Text; -using HDF.PInvoke; -using Tensorflow.NumPy; -using HDF5CSharp; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Tensorflow.Keras.Saving -{ - public class hdf5_format - { - private static int HDF5_OBJECT_HEADER_LIMIT = 64512; - public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) - { - long root = Hdf5.OpenFile(filepath,true); - load_model_from_hdf5(root, custom_objects, compile); - } - public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - //long fileId = filepath; - //try - //{ - // groupId = H5G.open(fileId, "/"); - // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); - // H5G.close(groupId); - // if (success == true) { - // Console.WriteLine(attrId[0]); - // } - //} - //catch (Exception ex) - //{ - // if (filepath != -1) { - // Hdf5.CloseFile(filepath); - // } - // if (groupId != -1) { - // H5G.close(groupId); - // } - // throw new Exception(ex.ToString()); - //} - - } - public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - /// - /// Preprocess layer weights between different Keras formats. - /// - /// - /// - /// - /// - public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) - { - // convert CuDNN layers - return _convert_rnn_weights(layer, weights); - } - - /// - /// Converts weights for RNN layers between native and CuDNN format. - /// - /// - /// - static List _convert_rnn_weights(ILayer layer, List weights) - { - var target_class = layer.GetType().Name; - return weights; - } - - public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group(long f, List layers) - { - string original_keras_version = "2.5.0"; - string original_backend = null; - var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); - if (success) - original_keras_version = attr.First(); - // keras version should be 2.5.0+ - var ver_major = int.Parse(original_keras_version.Split('.')[0]); - var ver_minor = int.Parse(original_keras_version.Split('.')[1]); - if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) - throw new ValueError("keras version should be 2.5.0 or later."); - - (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); - if (success) - original_backend = attr.First(); - - var filtered_layers = new List(); - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count > 0) - filtered_layers.append(layer); - } - - string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); - var filtered_layer_names = new List(); - foreach(var name in layer_names) - { - if (!filtered_layers.Select(x => x.Name).Contains(name)) - continue; - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - if (weight_names.Count() > 0) - filtered_layer_names.Add(name); - H5G.close(g); - } - - layer_names = filtered_layer_names.ToArray(); - if (layer_names.Length != filtered_layers.Count()) - throw new ValueError("You are trying to load a weight file " + - $"containing {layer_names}" + - $" layers into a model with {filtered_layers.Count} layers."); - - var weight_value_tuples = new List<(IVariableV1, NDArray)>(); - foreach (var (k, name) in enumerate(layer_names)) - { - var weight_values = new List(); - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - foreach (var i_ in weight_names) - { - (success, Array result) = Hdf5.ReadDataset(g, i_); - if (success) - weight_values.Add(np.array(result)); - } - H5G.close(g); - var layer = filtered_layers[k]; - var symbolic_weights = _legacy_weights(layer); - preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); - if (weight_values.Count() != symbolic_weights.Count()) - throw new ValueError($"Layer #{k} (named {layer.Name}" + - "in the current model) was found to " + - $"correspond to layer {name} in the save file." + - $"However the new layer {layer.Name} expects " + - $"{symbolic_weights.Count()} weights, but the saved weights have " + - $"{weight_values.Count()} elements."); - weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); - } - - keras.backend.batch_set_value(weight_value_tuples); - } - - public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void save_weights_to_hdf5_group(long f, List layers) - { - List layerName=new List(); - foreach (var layer in layers) - { - layerName.Add(layer.Name); - } - save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); - Hdf5.WriteAttribute(f, "backend", "tensorflow"); - Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); - - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count == 0) - continue; - - var weight_names = new List(); - // weight_values= keras.backend.batch_get_value(weights); - foreach (var weight in weights) - weight_names.Add(weight.Name); - - var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); - save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); - foreach (var (name, val) in zip(weight_names, weights)) - { - var tensor = val.AsTensor(); - if (name.IndexOf("/") > 1) - { - var crDataGroup = g; - string[] name_split = name.Split('/'); - for(int i = 0; i < name_split.Length; i++) - { - if (i == name_split.Length - 1) break; +using System; +using System.Collections.Generic; +using System.Text; +using HDF.PInvoke; +using Tensorflow.NumPy; +using HDF5CSharp; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tensorflow.Keras.Saving +{ + public class hdf5_format + { + private static int HDF5_OBJECT_HEADER_LIMIT = 64512; + public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) + { + long root = Hdf5.OpenFile(filepath,true); + load_model_from_hdf5(root, custom_objects, compile); + } + public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + //long fileId = filepath; + //try + //{ + // groupId = H5G.open(fileId, "/"); + // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); + // H5G.close(groupId); + // if (success == true) { + // Console.WriteLine(attrId[0]); + // } + //} + //catch (Exception ex) + //{ + // if (filepath != -1) { + // Hdf5.CloseFile(filepath); + // } + // if (groupId != -1) { + // H5G.close(groupId); + // } + // throw new Exception(ex.ToString()); + //} + + } + public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + /// + /// Preprocess layer weights between different Keras formats. + /// + /// + /// + /// + /// + public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) + { + // convert CuDNN layers + return _convert_rnn_weights(layer, weights); + } + + /// + /// Converts weights for RNN layers between native and CuDNN format. + /// + /// + /// + static List _convert_rnn_weights(ILayer layer, List weights) + { + var target_class = layer.GetType().Name; + return weights; + } + + public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group(long f, List layers) + { + string original_keras_version = "2.5.0"; + string original_backend = null; + var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); + if (success) + original_keras_version = attr.First(); + // keras version should be 2.5.0+ + var ver_major = int.Parse(original_keras_version.Split('.')[0]); + var ver_minor = int.Parse(original_keras_version.Split('.')[1]); + if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) + throw new ValueError("keras version should be 2.5.0 or later."); + + (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); + if (success) + original_backend = attr.First(); + + var filtered_layers = new List(); + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count > 0) + filtered_layers.append(layer); + } + + string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); + var filtered_layer_names = new List(); + foreach(var name in layer_names) + { + if (!filtered_layers.Select(x => x.Name).Contains(name)) + continue; + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + if (weight_names.Count() > 0) + filtered_layer_names.Add(name); + H5G.close(g); + } + + layer_names = filtered_layer_names.ToArray(); + if (layer_names.Length != filtered_layers.Count()) + throw new ValueError("You are trying to load a weight file " + + $"containing {layer_names}" + + $" layers into a model with {filtered_layers.Count} layers."); + + var weight_value_tuples = new List<(IVariableV1, NDArray)>(); + foreach (var (k, name) in enumerate(layer_names)) + { + var weight_values = new List(); + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + foreach (var i_ in weight_names) + { + (success, Array result) = Hdf5.ReadDataset(g, i_); + if (success) + weight_values.Add(np.array(result)); + } + H5G.close(g); + var layer = filtered_layers[k]; + var symbolic_weights = _legacy_weights(layer); + preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); + if (weight_values.Count() != symbolic_weights.Count()) + throw new ValueError($"Layer #{k} (named {layer.Name}" + + "in the current model) was found to " + + $"correspond to layer {name} in the save file." + + $"However the new layer {layer.Name} expects " + + $"{symbolic_weights.Count()} weights, but the saved weights have " + + $"{weight_values.Count()} elements."); + weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); + } + + keras.backend.batch_set_value(weight_value_tuples); + } + + public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void save_weights_to_hdf5_group(long f, List layers) + { + List layerName=new List(); + foreach (var layer in layers) + { + layerName.Add(layer.Name); + } + save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); + Hdf5.WriteAttribute(f, "backend", "tensorflow"); + Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); + + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count == 0) + continue; + + var weight_names = new List(); + // weight_values= keras.backend.batch_get_value(weights); + foreach (var weight in weights) + weight_names.Add(weight.Name); + + var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); + save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); + foreach (var (name, val) in zip(weight_names, weights)) + { + var tensor = val.AsTensor(); + if (name.IndexOf("/") > 1) + { + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length - 1; i++) + { crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); - } - WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); - Hdf5.CloseGroup(crDataGroup); - } - else - { - WriteDataset(g, name, tensor); - } - } - Hdf5.CloseGroup(g); - } - } - - private static void save_attributes_to_hdf5_group(long f, string name, Array data) - { - int num_chunks = 1; - - var chunked_data = Split(data, num_chunks); - int getSize = 0; - - string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; - - switch (getType) - { - case "single": - getSize = sizeof(float); - break; - case "double": - getSize = sizeof(double); - break; - case "string": - getSize = -1; - break; - case "int32": - getSize = sizeof(int); - break; - case "int64": - getSize = sizeof(long); - break; - default: - getSize = -1; - break; - } - int getCount = chunked_data.Count; - - if (getSize != -1) - { - num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); - if (num_chunks > 1) chunked_data = Split(data, num_chunks); - } - - if (num_chunks > 1) - { - foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) - WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); - } - else - { - WriteAttrs(f, getType, name, data); - } - } - - private static void WriteDataset(long f, string name, Tensor data) - { - switch (data.dtype) - { - case TF_DataType.TF_FLOAT: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_DOUBLE: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT32: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT64: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - default: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - } - } - - private static void WriteAttrs(long f,string typename, string name, Array data) - { - switch (typename) - { - case "single": - Hdf5.WriteAttributes(f, name, data); - break; - case "double": - Hdf5.WriteAttributes(f, name, data); - break; - case "string": - Hdf5.WriteAttributes(f, name, data); - break; - case "int32": - Hdf5.WriteAttributes(f, name, data); - break; - case "int64": - Hdf5.WriteAttributes(f, name, data); - break; - default: - Hdf5.WriteAttributes(f, name,data); - break; - } - } - - private static List> Split(Array list, int chunkSize) - { - var splitList = new List>(); - var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); - - for (int c = 0; c < chunkCount; c++) - { - var skip = c * chunkSize; - var take = skip + chunkSize; - var chunk = new List(chunkSize); - - for (int e = skip; e < take && e < list.Length; e++) - { - chunk.Add(list.GetValue(e)); - } - splitList.Add(chunk); - } - - return splitList; - } - - public static string[] load_attributes_from_hdf5_group(long group, string name) - { - var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); - if (success) - return attr.ToArray(); - - return null; - } - - public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static List _legacy_weights(ILayer layer) - { - var weights = layer.TrainableWeights.Select(x => x).ToList(); - weights.AddRange(layer.NonTrainableWeights); - return weights; - } - } -} - + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); + Hdf5.CloseGroup(crDataGroup); + } + else + { + WriteDataset(g, name, tensor); + } + } + Hdf5.CloseGroup(g); + } + } + + private static void save_attributes_to_hdf5_group(long f, string name, Array data) + { + int num_chunks = 1; + + var chunked_data = Split(data, num_chunks); + int getSize = 0; + + string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; + + switch (getType) + { + case "single": + getSize = sizeof(float); + break; + case "double": + getSize = sizeof(double); + break; + case "string": + getSize = -1; + break; + case "int32": + getSize = sizeof(int); + break; + case "int64": + getSize = sizeof(long); + break; + default: + getSize = -1; + break; + } + int getCount = chunked_data.Count; + + if (getSize != -1) + { + num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); + if (num_chunks > 1) chunked_data = Split(data, num_chunks); + } + + if (num_chunks > 1) + { + foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) + WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); + } + else + { + WriteAttrs(f, getType, name, data); + } + } + + private static void WriteDataset(long f, string name, Tensor data) + { + switch (data.dtype) + { + case TF_DataType.TF_FLOAT: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_DOUBLE: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT32: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT64: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + default: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + } + } + + private static void WriteAttrs(long f,string typename, string name, Array data) + { + switch (typename) + { + case "single": + Hdf5.WriteAttributes(f, name, data); + break; + case "double": + Hdf5.WriteAttributes(f, name, data); + break; + case "string": + Hdf5.WriteAttributes(f, name, data); + break; + case "int32": + Hdf5.WriteAttributes(f, name, data); + break; + case "int64": + Hdf5.WriteAttributes(f, name, data); + break; + default: + Hdf5.WriteAttributes(f, name,data); + break; + } + } + + private static List> Split(Array list, int chunkSize) + { + var splitList = new List>(); + var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); + + for (int c = 0; c < chunkCount; c++) + { + var skip = c * chunkSize; + var take = skip + chunkSize; + var chunk = new List(chunkSize); + + for (int e = skip; e < take && e < list.Length; e++) + { + chunk.Add(list.GetValue(e)); + } + splitList.Add(chunk); + } + + return splitList; + } + + public static string[] load_attributes_from_hdf5_group(long group, string name) + { + var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); + if (success) + return attr.ToArray(); + + return null; + } + + public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static List _legacy_weights(ILayer layer) + { + var weights = layer.TrainableWeights.Select(x => x).ToList(); + weights.AddRange(layer.NonTrainableWeights); + return weights; + } + } +} + From 8b17b14f30e288705552a5ca417264b35b8447bc Mon Sep 17 00:00:00 2001 From: Beacontownfc <89081023+Beacontownfc@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:34:08 +0800 Subject: [PATCH 66/77] Update hdf5_format.cs --- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 708 +++++++++--------- 1 file changed, 354 insertions(+), 354 deletions(-) diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index c80f653f..bab0efec 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -1,354 +1,354 @@ -using System; -using System.Collections.Generic; -using System.Text; -using HDF.PInvoke; -using Tensorflow.NumPy; -using HDF5CSharp; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; -using System.Linq; -using System.Text.RegularExpressions; - -namespace Tensorflow.Keras.Saving -{ - public class hdf5_format - { - private static int HDF5_OBJECT_HEADER_LIMIT = 64512; - public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) - { - long root = Hdf5.OpenFile(filepath,true); - load_model_from_hdf5(root, custom_objects, compile); - } - public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - //long fileId = filepath; - //try - //{ - // groupId = H5G.open(fileId, "/"); - // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); - // H5G.close(groupId); - // if (success == true) { - // Console.WriteLine(attrId[0]); - // } - //} - //catch (Exception ex) - //{ - // if (filepath != -1) { - // Hdf5.CloseFile(filepath); - // } - // if (groupId != -1) { - // H5G.close(groupId); - // } - // throw new Exception(ex.ToString()); - //} - - } - public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - /// - /// Preprocess layer weights between different Keras formats. - /// - /// - /// - /// - /// - public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) - { - // convert CuDNN layers - return _convert_rnn_weights(layer, weights); - } - - /// - /// Converts weights for RNN layers between native and CuDNN format. - /// - /// - /// - static List _convert_rnn_weights(ILayer layer, List weights) - { - var target_class = layer.GetType().Name; - return weights; - } - - public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group(long f, List layers) - { - string original_keras_version = "2.5.0"; - string original_backend = null; - var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); - if (success) - original_keras_version = attr.First(); - // keras version should be 2.5.0+ - var ver_major = int.Parse(original_keras_version.Split('.')[0]); - var ver_minor = int.Parse(original_keras_version.Split('.')[1]); - if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) - throw new ValueError("keras version should be 2.5.0 or later."); - - (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); - if (success) - original_backend = attr.First(); - - var filtered_layers = new List(); - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count > 0) - filtered_layers.append(layer); - } - - string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); - var filtered_layer_names = new List(); - foreach(var name in layer_names) - { - if (!filtered_layers.Select(x => x.Name).Contains(name)) - continue; - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - if (weight_names.Count() > 0) - filtered_layer_names.Add(name); - H5G.close(g); - } - - layer_names = filtered_layer_names.ToArray(); - if (layer_names.Length != filtered_layers.Count()) - throw new ValueError("You are trying to load a weight file " + - $"containing {layer_names}" + - $" layers into a model with {filtered_layers.Count} layers."); - - var weight_value_tuples = new List<(IVariableV1, NDArray)>(); - foreach (var (k, name) in enumerate(layer_names)) - { - var weight_values = new List(); - long g = H5G.open(f, name); - var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); - foreach (var i_ in weight_names) - { - (success, Array result) = Hdf5.ReadDataset(g, i_); - if (success) - weight_values.Add(np.array(result)); - } - H5G.close(g); - var layer = filtered_layers[k]; - var symbolic_weights = _legacy_weights(layer); - preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); - if (weight_values.Count() != symbolic_weights.Count()) - throw new ValueError($"Layer #{k} (named {layer.Name}" + - "in the current model) was found to " + - $"correspond to layer {name} in the save file." + - $"However the new layer {layer.Name} expects " + - $"{symbolic_weights.Count()} weights, but the saved weights have " + - $"{weight_values.Count()} elements."); - weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); - } - - keras.backend.batch_set_value(weight_value_tuples); - } - - public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static void save_weights_to_hdf5_group(long f, List layers) - { - List layerName=new List(); - foreach (var layer in layers) - { - layerName.Add(layer.Name); - } - save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); - Hdf5.WriteAttribute(f, "backend", "tensorflow"); - Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); - - foreach (var layer in layers) - { - var weights = _legacy_weights(layer); - if (weights.Count == 0) - continue; - - var weight_names = new List(); - // weight_values= keras.backend.batch_get_value(weights); - foreach (var weight in weights) - weight_names.Add(weight.Name); - - var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); - save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); - foreach (var (name, val) in zip(weight_names, weights)) - { - var tensor = val.AsTensor(); - if (name.IndexOf("/") > 1) - { - var crDataGroup = g; - string[] name_split = name.Split('/'); - for(int i = 0; i < name_split.Length - 1; i++) - { - crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); - } - WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); - Hdf5.CloseGroup(crDataGroup); - } - else - { - WriteDataset(g, name, tensor); - } - } - Hdf5.CloseGroup(g); - } - } - - private static void save_attributes_to_hdf5_group(long f, string name, Array data) - { - int num_chunks = 1; - - var chunked_data = Split(data, num_chunks); - int getSize = 0; - - string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; - - switch (getType) - { - case "single": - getSize = sizeof(float); - break; - case "double": - getSize = sizeof(double); - break; - case "string": - getSize = -1; - break; - case "int32": - getSize = sizeof(int); - break; - case "int64": - getSize = sizeof(long); - break; - default: - getSize = -1; - break; - } - int getCount = chunked_data.Count; - - if (getSize != -1) - { - num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); - if (num_chunks > 1) chunked_data = Split(data, num_chunks); - } - - if (num_chunks > 1) - { - foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) - WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); - } - else - { - WriteAttrs(f, getType, name, data); - } - } - - private static void WriteDataset(long f, string name, Tensor data) - { - switch (data.dtype) - { - case TF_DataType.TF_FLOAT: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_DOUBLE: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT32: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - case TF_DataType.TF_INT64: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - default: - Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); - break; - } - } - - private static void WriteAttrs(long f,string typename, string name, Array data) - { - switch (typename) - { - case "single": - Hdf5.WriteAttributes(f, name, data); - break; - case "double": - Hdf5.WriteAttributes(f, name, data); - break; - case "string": - Hdf5.WriteAttributes(f, name, data); - break; - case "int32": - Hdf5.WriteAttributes(f, name, data); - break; - case "int64": - Hdf5.WriteAttributes(f, name, data); - break; - default: - Hdf5.WriteAttributes(f, name,data); - break; - } - } - - private static List> Split(Array list, int chunkSize) - { - var splitList = new List>(); - var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); - - for (int c = 0; c < chunkCount; c++) - { - var skip = c * chunkSize; - var take = skip + chunkSize; - var chunk = new List(chunkSize); - - for (int e = skip; e < take && e < list.Length; e++) - { - chunk.Add(list.GetValue(e)); - } - splitList.Add(chunk); - } - - return splitList; - } - - public static string[] load_attributes_from_hdf5_group(long group, string name) - { - var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); - if (success) - return attr.ToArray(); - - return null; - } - - public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) - { - - } - - public static List _legacy_weights(ILayer layer) - { - var weights = layer.TrainableWeights.Select(x => x).ToList(); - weights.AddRange(layer.NonTrainableWeights); - return weights; - } - } -} - +using System; +using System.Collections.Generic; +using System.Text; +using HDF.PInvoke; +using Tensorflow.NumPy; +using HDF5CSharp; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tensorflow.Keras.Saving +{ + public class hdf5_format + { + private static int HDF5_OBJECT_HEADER_LIMIT = 64512; + public static void load_model_from_hdf5(string filepath = "", Dictionary custom_objects = null, bool compile = false) + { + long root = Hdf5.OpenFile(filepath,true); + load_model_from_hdf5(root, custom_objects, compile); + } + public static void load_model_from_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + //long fileId = filepath; + //try + //{ + // groupId = H5G.open(fileId, "/"); + // (bool success, string[] attrId) = Hdf5.ReadStringAttributes(groupId, "model_config", ""); + // H5G.close(groupId); + // if (success == true) { + // Console.WriteLine(attrId[0]); + // } + //} + //catch (Exception ex) + //{ + // if (filepath != -1) { + // Hdf5.CloseFile(filepath); + // } + // if (groupId != -1) { + // H5G.close(groupId); + // } + // throw new Exception(ex.ToString()); + //} + + } + public static void save_model_to_hdf5(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + /// + /// Preprocess layer weights between different Keras formats. + /// + /// + /// + /// + /// + public static List preprocess_weights_for_loading(ILayer layer, List weights, string original_keras_version = null, string original_backend = null) + { + // convert CuDNN layers + return _convert_rnn_weights(layer, weights); + } + + /// + /// Converts weights for RNN layers between native and CuDNN format. + /// + /// + /// + static List _convert_rnn_weights(ILayer layer, List weights) + { + var target_class = layer.GetType().Name; + return weights; + } + + public static void save_optimizer_weights_to_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_optimizer_weights_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group(long f, List layers) + { + string original_keras_version = "2.5.0"; + string original_backend = null; + var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); + if (success) + original_keras_version = attr.First(); + // keras version should be 2.5.0+ + var ver_major = int.Parse(original_keras_version.Split('.')[0]); + var ver_minor = int.Parse(original_keras_version.Split('.')[1]); + if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) + throw new ValueError("keras version should be 2.5.0 or later."); + + (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); + if (success) + original_backend = attr.First(); + + var filtered_layers = new List(); + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count > 0) + filtered_layers.append(layer); + } + + string[] layer_names = load_attributes_from_hdf5_group(f, "layer_names"); + var filtered_layer_names = new List(); + foreach(var name in layer_names) + { + if (!filtered_layers.Select(x => x.Name).Contains(name)) + continue; + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + if (weight_names.Count() > 0) + filtered_layer_names.Add(name); + H5G.close(g); + } + + layer_names = filtered_layer_names.ToArray(); + if (layer_names.Length != filtered_layers.Count()) + throw new ValueError("You are trying to load a weight file " + + $"containing {layer_names}" + + $" layers into a model with {filtered_layers.Count} layers."); + + var weight_value_tuples = new List<(IVariableV1, NDArray)>(); + foreach (var (k, name) in enumerate(layer_names)) + { + var weight_values = new List(); + long g = H5G.open(f, name); + var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); + foreach (var i_ in weight_names) + { + (success, Array result) = Hdf5.ReadDataset(g, i_); + if (success) + weight_values.Add(np.array(result)); + } + H5G.close(g); + var layer = filtered_layers[k]; + var symbolic_weights = _legacy_weights(layer); + preprocess_weights_for_loading(layer, weight_values, original_keras_version, original_backend); + if (weight_values.Count() != symbolic_weights.Count()) + throw new ValueError($"Layer #{k} (named {layer.Name}" + + "in the current model) was found to " + + $"correspond to layer {name} in the save file." + + $"However the new layer {layer.Name} expects " + + $"{symbolic_weights.Count()} weights, but the saved weights have " + + $"{weight_values.Count()} elements."); + weight_value_tuples.AddRange(zip(symbolic_weights, weight_values)); + } + + keras.backend.batch_set_value(weight_value_tuples); + } + + public static void toarrayf4(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void load_weights_from_hdf5_group_by_name(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static void save_weights_to_hdf5_group(long f, List layers) + { + List layerName=new List(); + foreach (var layer in layers) + { + layerName.Add(layer.Name); + } + save_attributes_to_hdf5_group(f, "layer_names", layerName.ToArray()); + Hdf5.WriteAttribute(f, "backend", "tensorflow"); + Hdf5.WriteAttribute(f, "keras_version", "2.5.0"); + + foreach (var layer in layers) + { + var weights = _legacy_weights(layer); + if (weights.Count == 0) + continue; + + var weight_names = new List(); + // weight_values= keras.backend.batch_get_value(weights); + foreach (var weight in weights) + weight_names.Add(weight.Name); + + var g = Hdf5.CreateOrOpenGroup(f, Hdf5Utils.NormalizedName(layer.Name)); + save_attributes_to_hdf5_group(g, "weight_names", weight_names.ToArray()); + foreach (var (name, val) in zip(weight_names, weights)) + { + var tensor = val.AsTensor(); + if (name.IndexOf("/") > 1) + { + var crDataGroup = g; + string[] name_split = name.Split('/'); + for(int i = 0; i < name_split.Length - 1; i++) + { + crDataGroup = Hdf5.CreateOrOpenGroup(crDataGroup, Hdf5Utils.NormalizedName(name_split[i])); + } + WriteDataset(crDataGroup, name_split[name_split.Length - 1], tensor); + Hdf5.CloseGroup(crDataGroup); + } + else + { + WriteDataset(g, name, tensor); + } + } + Hdf5.CloseGroup(g); + } + } + + private static void save_attributes_to_hdf5_group(long f, string name, Array data) + { + int num_chunks = 1; + + var chunked_data = Split(data, num_chunks); + int getSize = 0; + + string getType = data.Length > 0 ? data.GetValue(0).GetType().Name.ToLower() : "string"; + + switch (getType) + { + case "single": + getSize = sizeof(float); + break; + case "double": + getSize = sizeof(double); + break; + case "string": + getSize = -1; + break; + case "int32": + getSize = sizeof(int); + break; + case "int64": + getSize = sizeof(long); + break; + default: + getSize = -1; + break; + } + int getCount = chunked_data.Count; + + if (getSize != -1) + { + num_chunks = (int)Math.Ceiling((double)(getCount * getSize) / HDF5_OBJECT_HEADER_LIMIT); + if (num_chunks > 1) chunked_data = Split(data, num_chunks); + } + + if (num_chunks > 1) + { + foreach (var (chunk_id, chunk_data) in enumerate(chunked_data)) + WriteAttrs(f, getType, $"{name}{chunk_id}", chunk_data.ToArray()); + } + else + { + WriteAttrs(f, getType, name, data); + } + } + + private static void WriteDataset(long f, string name, Tensor data) + { + switch (data.dtype) + { + case TF_DataType.TF_FLOAT: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_DOUBLE: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT32: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + case TF_DataType.TF_INT64: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + default: + Hdf5.WriteDatasetFromArray(f, name, data.numpy().ToMultiDimArray()); + break; + } + } + + private static void WriteAttrs(long f,string typename, string name, Array data) + { + switch (typename) + { + case "single": + Hdf5.WriteAttributes(f, name, data); + break; + case "double": + Hdf5.WriteAttributes(f, name, data); + break; + case "string": + Hdf5.WriteAttributes(f, name, data); + break; + case "int32": + Hdf5.WriteAttributes(f, name, data); + break; + case "int64": + Hdf5.WriteAttributes(f, name, data); + break; + default: + Hdf5.WriteAttributes(f, name,data); + break; + } + } + + private static List> Split(Array list, int chunkSize) + { + var splitList = new List>(); + var chunkCount = (int)Math.Ceiling((double)list.Length / (double)chunkSize); + + for (int c = 0; c < chunkCount; c++) + { + var skip = c * chunkSize; + var take = skip + chunkSize; + var chunk = new List(chunkSize); + + for (int e = skip; e < take && e < list.Length; e++) + { + chunk.Add(list.GetValue(e)); + } + splitList.Add(chunk); + } + + return splitList; + } + + public static string[] load_attributes_from_hdf5_group(long group, string name) + { + var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); + if (success) + return attr.ToArray(); + + return null; + } + + public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionary custom_objects = null, bool compile = false) + { + + } + + public static List _legacy_weights(ILayer layer) + { + var weights = layer.TrainableWeights.Select(x => x).ToList(); + weights.AddRange(layer.NonTrainableWeights); + return weights; + } + } +} + From 482899eab734f1b6f3a39ef52a4f9ae28e332ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Sat, 22 Jul 2023 15:03:50 +0800 Subject: [PATCH 67/77] fix: revise np.amin, np.amax and add np.argmin --- .../NumPy/NumPy.Sorting.Searching.Counting.cs | 4 ++++ src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs | 4 ++-- src/TensorFlowNET.Core/Operations/math_ops.cs | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs index 5182d572..4cad36e0 100644 --- a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs +++ b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs @@ -13,6 +13,10 @@ namespace Tensorflow.NumPy public static NDArray argmax(NDArray a, Axis? axis = null) => new NDArray(math_ops.argmax(a, axis ?? 0)); + [AutoNumPy] + public static NDArray argmin(NDArray a, Axis? axis = null) + => new NDArray(math_ops.argmin(a, axis ?? 0)); + [AutoNumPy] public static NDArray argsort(NDArray a, Axis? axis = null) => new NDArray(sort_ops.argsort(a, axis: axis ?? -1)); diff --git a/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs b/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs index 5d86b1b3..bce16ec9 100644 --- a/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs +++ b/src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs @@ -10,10 +10,10 @@ namespace Tensorflow.NumPy public partial class np { [AutoNumPy] - public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.arg_min(x, axis)); + public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.min(x, axis)); [AutoNumPy] - public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.math.argmax(x, axis)); + public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.max(x, axis)); [AutoNumPy] public static NDArray average(NDArray a, int axis = -1, NDArray? weights = null, bool returned = false) diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 092137bf..e77df702 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -77,6 +77,9 @@ namespace Tensorflow public static Tensor argmax(Tensor input, Axis dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) => gen_math_ops.arg_max(input, dimension, output_type: output_type, name: name); + public static Tensor argmin(Tensor input, Axis dimension, TF_DataType output_type = TF_DataType.TF_INT64, string name = null) + => gen_math_ops.arg_min(input, dimension, output_type: output_type, name: name); + public static Tensor round(Tensor x, string name = null) { x = ops.convert_to_tensor(x, name: "x"); From b0ce73caff995d8b5b8080dd41812af4c48908e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Mon, 24 Jul 2023 23:38:58 +0800 Subject: [PATCH 68/77] feat: add adjust_contrast, adjust_hue, combined_non_max_suppression, crop_and_resize image oprs --- src/TensorFlowNET.Core/APIs/tf.image.cs | 131 +++++++- .../Operations/gen_image_ops.cs | 298 +++++++++++++++++- .../TensorFlowNET.Graph.UnitTest/ImageTest.cs | 65 +++- 3 files changed, 479 insertions(+), 15 deletions(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.image.cs b/src/TensorFlowNET.Core/APIs/tf.image.cs index 9230b50d..ac9cbc60 100644 --- a/src/TensorFlowNET.Core/APIs/tf.image.cs +++ b/src/TensorFlowNET.Core/APIs/tf.image.cs @@ -14,6 +14,10 @@ limitations under the License. ******************************************************************************/ +using OneOf.Types; +using System; +using System.Buffers.Text; +using Tensorflow.Contexts; using static Tensorflow.Binding; namespace Tensorflow @@ -162,17 +166,108 @@ namespace Tensorflow public Tensor sobel_edges(Tensor image) => image_ops_impl.sobel_edges(image); - public Tensor decode_jpeg(Tensor contents, - int channels = 0, - int ratio = 1, - bool fancy_upscaling = true, - bool try_recover_truncated = false, - int acceptable_fraction = 1, - string dct_method = "", - string name = null) - => gen_image_ops.decode_jpeg(contents, channels: channels, ratio: ratio, - fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, - acceptable_fraction: acceptable_fraction, dct_method: dct_method); + /// + /// Adjust contrast of RGB or grayscale images. + /// + /// Images to adjust. At least 3-D. + /// + /// A float multiplier for adjusting contrast. + /// The contrast-adjusted image or images. + public Tensor adjust_contrast(Tensor images, float contrast_factor, string name = null) + => gen_image_ops.adjust_contrastv2(images, contrast_factor, name); + + /// + /// Adjust hue of RGB images. + /// + /// RGB image or images. The size of the last dimension must be 3. + /// float. How much to add to the hue channel. + /// A name for this operation (optional). + /// Adjusted image(s), same shape and DType as `image`. + /// if `delta` is not in the interval of `[-1, 1]`. + public Tensor adjust_hue(Tensor images, float delta, string name = null) + { + if (tf.Context.executing_eagerly()) + { + if (delta < -1f || delta > 1f) + throw new ValueError("delta must be in the interval [-1, 1]"); + } + return gen_image_ops.adjust_hue(images, delta, name: name); + } + + /// + /// Adjust saturation of RGB images. + /// + /// RGB image or images. The size of the last dimension must be 3. + /// float. Factor to multiply the saturation by. + /// A name for this operation (optional). + /// Adjusted image(s), same shape and DType as `image`. + public Tensor adjust_saturation(Tensor image, float saturation_factor, string name = null) + => gen_image_ops.adjust_saturation(image, saturation_factor, name); + + /// + /// Greedily selects a subset of bounding boxes in descending order of score. + /// + /// + /// A 4-D float `Tensor` of shape `[batch_size, num_boxes, q, 4]`. If `q` + /// is 1 then same boxes are used for all classes otherwise, if `q` is equal + /// to number of classes, class-specific boxes are used. + /// + /// + /// A 3-D float `Tensor` of shape `[batch_size, num_boxes, num_classes]` + /// representing a single score corresponding to each box(each row of boxes). + /// + /// + /// A scalar integer `Tensor` representing the + /// maximum number of boxes to be selected by non-max suppression per class + /// + /// + /// A int32 scalar representing maximum number of boxes retained + /// over all classes.Note that setting this value to a large number may + /// result in OOM error depending on the system workload. + /// + /// + /// A float representing the threshold for deciding whether boxes + /// overlap too much with respect to IOU. + /// + /// + /// A float representing the threshold for deciding when to + /// remove boxes based on score. + /// + /// + /// If false, the output nmsed boxes, scores and classes are + /// padded/clipped to `max_total_size`. If true, the output nmsed boxes, scores and classes are padded to be of length `max_size_per_class`*`num_classes`, + /// unless it exceeds `max_total_size` in which case it is clipped to `max_total_size`. Defaults to false. + /// + /// + /// If true, the coordinates of output nmsed boxes will be clipped + /// to[0, 1]. If false, output the box coordinates as it is. Defaults to true. + /// + /// + /// 'nmsed_boxes': A [batch_size, max_detections, 4] float32 tensor containing the non-max suppressed boxes. + /// 'nmsed_scores': A [batch_size, max_detections] float32 tensor containing the scores for the boxes. + /// 'nmsed_classes': A [batch_size, max_detections] float32 tensor containing the class for boxes. + /// 'valid_detections': A [batch_size] int32 tensor indicating the number of + /// valid detections per batch item. Only the top valid_detections[i] entries + /// in nms_boxes[i], nms_scores[i] and nms_class[i] are valid. The rest of the + /// entries are zero paddings. + /// + public (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression( + Tensor boxes, + Tensor scores, + int max_output_size_per_class, + int max_total_size, + float iou_threshold, + float score_threshold, + bool pad_per_class = false, + bool clip_boxes = true) + { + var iou_threshold_t = ops.convert_to_tensor(iou_threshold, TF_DataType.TF_FLOAT, name: "iou_threshold"); + var score_threshold_t = ops.convert_to_tensor(score_threshold, TF_DataType.TF_FLOAT, name: "score_threshold"); + var max_total_size_t = ops.convert_to_tensor(max_total_size); + var max_output_size_per_class_t = ops.convert_to_tensor(max_output_size_per_class); + return gen_image_ops.combined_non_max_suppression(boxes, scores, max_output_size_per_class_t, max_total_size_t, + iou_threshold_t, score_threshold_t, pad_per_class, clip_boxes); + } /// /// Extracts crops from the input image tensor and resizes them using bilinear sampling or nearest neighbor sampling (possibly with aspect ratio change) to a common output size specified by crop_size. This is more general than the crop_to_bounding_box op which extracts a fixed size slice from the input image and does not allow resizing or aspect ratio change. @@ -187,7 +282,19 @@ namespace Tensorflow /// A name for the operation (optional). /// A 4-D tensor of shape [num_boxes, crop_height, crop_width, depth]. public Tensor crop_and_resize(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method = "bilinear", float extrapolation_value = 0f, string name = null) => - image_ops_impl.crop_and_resize(image, boxes, box_ind, crop_size, method, extrapolation_value, name); + gen_image_ops.crop_and_resize(image, boxes, box_ind, crop_size, method, extrapolation_value, name); + + public Tensor decode_jpeg(Tensor contents, + int channels = 0, + int ratio = 1, + bool fancy_upscaling = true, + bool try_recover_truncated = false, + int acceptable_fraction = 1, + string dct_method = "", + string name = null) + => gen_image_ops.decode_jpeg(contents, channels: channels, ratio: ratio, + fancy_upscaling: fancy_upscaling, try_recover_truncated: try_recover_truncated, + acceptable_fraction: acceptable_fraction, dct_method: dct_method); public Tensor extract_glimpse(Tensor input, Tensor size, Tensor offsets, bool centered = true, bool normalized = true, bool uniform_noise = true, string name = null) diff --git a/src/TensorFlowNET.Core/Operations/gen_image_ops.cs b/src/TensorFlowNET.Core/Operations/gen_image_ops.cs index 9240b590..cbe661ae 100644 --- a/src/TensorFlowNET.Core/Operations/gen_image_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_image_ops.cs @@ -16,18 +16,312 @@ using System; using System.Linq; +using Tensorflow.Eager; using static Tensorflow.Binding; +using Tensorflow.Exceptions; +using Tensorflow.Contexts; +using System.Xml.Linq; +using Google.Protobuf; namespace Tensorflow { public class gen_image_ops { + public static Tensor adjust_contrastv2(Tensor images, Tensor contrast_factor, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustContrastv2", name) { + args = new object[] { images, contrast_factor }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_contrastv2_eager_fallback(images, contrast_factor, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["contrast_factor"] = contrast_factor; + var _op = tf.OpDefLib._apply_op_helper("AdjustContrastv2", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustContrastv2", _op.inputs, _attrs, _result); + } + return _result[0]; + } + public static Tensor adjust_contrastv2(Tensor image, float contrast_factor, string name = null) + { + return adjust_contrastv2(image, tf.convert_to_tensor(contrast_factor), name: name); + } + + public static Tensor adjust_contrastv2_eager_fallback(Tensor images, Tensor contrast_factor, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, contrast_factor}; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustContrastv2", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustContrastv2", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_hue(Tensor images, Tensor delta, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustHue", name) { + args = new object[] { images, delta }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_hue_eager_fallback(images, delta, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["delta"] = delta; + var _op = tf.OpDefLib._apply_op_helper("AdjustHue", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustHue", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_hue(Tensor images, float delta, string name = null) + => adjust_hue(images, delta, name: name); + + public static Tensor adjust_hue_eager_fallback(Tensor images, Tensor delta, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, delta}; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustHue", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustHue", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_saturation(Tensor images, Tensor scale, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "AdjustSaturation", name) + { + args = new object[] { images, scale }, + attrs = new Dictionary() { } + }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return adjust_hue_eager_fallback(images, scale, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["images"] = images; + keywords["scale"] = scale; + var _op = tf.OpDefLib._apply_op_helper("AdjustSaturation", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") }; + _execute.record_gradient("AdjustSaturation", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor adjust_saturation(Tensor images, float scale, string name = null) + => adjust_saturation(images, ops.convert_to_tensor(scale), name: name); + + public static Tensor adjust_saturation_eager_fallback(Tensor images, Tensor scale, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { images, scale }; + object[] _attrs = new object[] { "T", images.dtype }; + var _result = _execute.execute("AdjustSaturation", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("AdjustSaturation", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + public static (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression(Tensor boxes, Tensor scores, Tensor max_output_size_per_class, Tensor max_total_size, - Tensor iou_threshold, Tensor score_threshold, bool pad_per_class, bool clip_boxes) + Tensor iou_threshold, Tensor score_threshold, bool pad_per_class = false, bool clip_boxes = true, string name = null) { - throw new NotImplementedException("combined_non_max_suppression"); + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CombinedNonMaxSuppression", name){ + args = new object[] { + boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold, + "pad_per_class", pad_per_class, "clip_boxes", clip_boxes}, + attrs = new Dictionary() { }}); + return (_fast_path_result[0], _fast_path_result[1], _fast_path_result[2], _fast_path_result[3]); + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return combined_non_max_suppression_eager_fallback( + boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, + score_threshold, pad_per_class, clip_boxes, name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["boxes"] = boxes; + keywords["scores"] = scores; + keywords["max_output_size_per_class"] = max_output_size_per_class; + keywords["max_total_size"] = max_total_size; + keywords["iou_threshold"] = iou_threshold; + keywords["score_threshold"] = score_threshold; + keywords["pad_per_class"] = pad_per_class; + keywords["clip_boxes"] = clip_boxes; + + var _op = tf.OpDefLib._apply_op_helper("CombinedNonMaxSuppression", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "pad_per_class", _op._get_attr_type("pad_per_class") ,"clip_boxes", _op._get_attr_type("clip_boxes")}; + _execute.record_gradient("CombinedNonMaxSuppression", _op.inputs, _attrs, _result); + } + return (_result[0], _result[1], _result[2], _result[3]); } + public static (Tensor, Tensor, Tensor, Tensor) combined_non_max_suppression_eager_fallback(Tensor boxes, Tensor scores, Tensor max_output_size_per_class, Tensor max_total_size, + Tensor iou_threshold, Tensor score_threshold, bool pad_per_class, bool clip_boxes, string name, Context ctx) + { + Tensor[] _inputs_flat = new Tensor[] { boxes, scores, max_output_size_per_class, max_total_size, iou_threshold, score_threshold }; + object[] _attrs = new object[] { "pad_per_class", pad_per_class, "clip_boxes", clip_boxes }; + var _result = _execute.execute("CombinedNonMaxSuppression", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("CombinedNonMaxSuppression", _inputs_flat, _attrs, _result); + } + return (_result[0], _result[1], _result[2], _result[3]); + } + + public static Tensor crop_and_resize(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method = "bilinear", float extrapolation_value = 0f, string name = null) + { + var _ctx = tf.Context; + if (_ctx.executing_eagerly()) + { + try + { + var _fast_path_result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo(_ctx, "CropAndResize", name) { + args = new object[] { + image, boxes, box_ind, crop_size, "method", method, "extrapolation_value", extrapolation_value }, attrs = new Dictionary() { } }); + return _fast_path_result[0]; + } + catch (NotOkStatusException ex) + { + throw ex; + } + catch (Exception) + { + } + try + { + return crop_and_resize_eager_fallback( + image, boxes, box_ind, crop_size, method: method, extrapolation_value: extrapolation_value, name: name, ctx: _ctx); + } + catch (Exception) + { + } + } + Dictionary keywords = new(); + keywords["image"] = image; + keywords["boxes"] = boxes; + keywords["box_ind"] = box_ind; + keywords["crop_size"] = crop_size; + keywords["method"] = method; + keywords["extrapolation_value"] = extrapolation_value; + var _op = tf.OpDefLib._apply_op_helper("CropAndResize", name, keywords); + var _result = _op.outputs; + if (_execute.must_record_gradient()) + { + object[] _attrs = new object[] { "T", _op._get_attr_type("T") ,"method", _op._get_attr_type("method") , + "extrapolation_value", _op.get_attr("extrapolation_value")}; + _execute.record_gradient("CropAndResize", _op.inputs, _attrs, _result); + } + return _result[0]; + } + + public static Tensor crop_and_resize_eager_fallback(Tensor image, Tensor boxes, Tensor box_ind, Tensor crop_size, string method, float extrapolation_value, string name, Context ctx) + { + if (method is null) + method = "bilinear"; + //var method_cpmpat = ByteString.CopyFromUtf8(method ?? string.Empty); + //var extrapolation_value_float = (float)extrapolation_value; + + Tensor[] _inputs_flat = new Tensor[] { image, boxes, box_ind, crop_size, tf.convert_to_tensor(method), tf.convert_to_tensor(extrapolation_value) }; + object[] _attrs = new object[] { "T", image.dtype }; + var _result = _execute.execute("CropAndResize", 1, inputs: _inputs_flat, attrs: _attrs, ctx: ctx, name: name); + if (_execute.must_record_gradient()) + { + _execute.record_gradient("CropAndResize", _inputs_flat, _attrs, _result); + } + return _result[0]; + } + + public static Tensor convert_image_dtype(Tensor image, TF_DataType dtype, bool saturate = false, string name = null) { if (dtype == image.dtype) diff --git a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs index c42445cf..151ea834 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs @@ -3,6 +3,7 @@ using Tensorflow.NumPy; using System.Linq; using Tensorflow; using static Tensorflow.Binding; +using System; namespace TensorFlowNET.UnitTest { @@ -22,13 +23,75 @@ namespace TensorFlowNET.UnitTest contents = tf.io.read_file(imgPath); } + [TestMethod] + public void adjust_contrast() + { + var input = np.array(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f); + var image = tf.reshape(input, new int[] { 3, 3, 1 }); + var img = tf.image.adjust_contrast(image, 2.0f); + var res = np.array(-4f, -2f, 0f, 2f, 4f, 6f, 8f, 10f, 12f).reshape((3,3,1)); + Assert.AreEqual(img.numpy(), res); + } + + [Ignore] + [TestMethod] + public void adjust_hue() + { + var image = tf.constant(new int[] {1,2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18}); + image = tf.reshape(image, new int[] { 3, 2, 3 }); + var adjusted_image = tf.image.adjust_hue(image, 0.2f); + var res = tf.constant(new int[] {2,1,3, 4, 5, 6,8,7,9,11,10,12,14,13,15,17,16,18}); + res = tf.reshape(res,(3,2,3)); + Assert.AreEqual(adjusted_image, res); + } + + [TestMethod] + public void combined_non_max_suppression() + { + var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 },{ 210, 112, 150, 100 } }); + var boxes1 = tf.reshape(boxesX, (1, 4, 1, 4)); + var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f },{ 0.1f, 0.8f, 0.1f },{ 0.3f, 0.6f, 0.1f },{ 0.05f, 0.9f, 0.05f } }); + var scores1 = tf.reshape(scoresX, (1, 4, 3)); + var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes:false); + + var boxes_gt = tf.constant(new float[,] { { 210f, 112f, 150f, 100f }, { 200f, 100f, 150f, 100f }, { 190f, 110f, 150f, 100f }, + { 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f , 0f},{ 0f, 0f, 0f, 0f},{ 0f , 0f, 0f, 0f},{ 0f, 0f, 0f, 0f} }); + boxes_gt = tf.reshape(boxes_gt,(1, 10, 4)); + Assert.AreEqual(boxes.numpy(), boxes_gt.numpy()); + var scores_gt = tf.constant(new float[,] { { 0.9f, 0.7f, 0.3f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); + scores_gt = tf.reshape(scores_gt, (1, 10)); + Assert.AreEqual(scores.numpy(), scores_gt.numpy()); + var classes_gt = tf.constant(new float[,] { { 1f, 1f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); + classes_gt = tf.reshape(classes_gt, (1, 10)); + Assert.AreEqual(classes.numpy(), classes_gt.numpy()); + var valid_detections_gt = tf.constant(new int[,] { { 3 } }); + valid_detections_gt = tf.reshape(valid_detections_gt, (1)); + Assert.AreEqual(valid_detections.numpy(), valid_detections_gt.numpy()); + } + + [TestMethod] + public void crop_and_resize() + { + int BATCH_SIZE = 1; + int NUM_BOXES = 5; + int IMAGE_HEIGHT = 256; + int IMAGE_WIDTH = 256; + int CHANNELS = 3; + var crop_size = tf.constant(new int[] { 24, 24 }); + var image = tf.random.uniform((BATCH_SIZE, IMAGE_HEIGHT, IMAGE_WIDTH, CHANNELS)); + var boxes = tf.random.uniform((NUM_BOXES, 4)); + var box_ind = tf.random.uniform((NUM_BOXES), minval: 0, maxval: BATCH_SIZE, dtype: TF_DataType.TF_INT32); + var output = tf.image.crop_and_resize(image, boxes, box_ind, crop_size); + Assert.AreEqual((5,24,24,3), output.shape); + } + [TestMethod] public void decode_image() { var img = tf.image.decode_image(contents); Assert.AreEqual(img.name, "decode_image/DecodeImage:0"); } - + [TestMethod] public void resize_image() { From 3273cbc7f2e14eb030dfc9967ce5bf550186a93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Tue, 25 Jul 2023 00:09:50 +0800 Subject: [PATCH 69/77] fix: fix ci error --- .../TensorFlowNET.Graph.UnitTest/ImageTest.cs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs index 151ea834..d671b609 100644 --- a/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs +++ b/test/TensorFlowNET.Graph.UnitTest/ImageTest.cs @@ -28,9 +28,14 @@ namespace TensorFlowNET.UnitTest { var input = np.array(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f); var image = tf.reshape(input, new int[] { 3, 3, 1 }); - var img = tf.image.adjust_contrast(image, 2.0f); + + var init = tf.global_variables_initializer(); + var sess = tf.Session(); + sess.run(init); + var adjust_contrast = tf.image.adjust_contrast(image, 2.0f); + var result = sess.run(adjust_contrast); var res = np.array(-4f, -2f, 0f, 2f, 4f, 6f, 8f, 10f, 12f).reshape((3,3,1)); - Assert.AreEqual(img.numpy(), res); + Assert.AreEqual(result.numpy(), res); } [Ignore] @@ -48,25 +53,31 @@ namespace TensorFlowNET.UnitTest [TestMethod] public void combined_non_max_suppression() { - var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 },{ 210, 112, 150, 100 } }); + var boxesX = tf.constant(new float[,] { { 200, 100, 150, 100 }, { 220, 120, 150, 100 }, { 190, 110, 150, 100 }, { 210, 112, 150, 100 } }); var boxes1 = tf.reshape(boxesX, (1, 4, 1, 4)); - var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f },{ 0.1f, 0.8f, 0.1f },{ 0.3f, 0.6f, 0.1f },{ 0.05f, 0.9f, 0.05f } }); + var scoresX = tf.constant(new float[,] { { 0.2f, 0.7f, 0.1f }, { 0.1f, 0.8f, 0.1f }, { 0.3f, 0.6f, 0.1f }, { 0.05f, 0.9f, 0.05f } }); var scores1 = tf.reshape(scoresX, (1, 4, 3)); - var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes:false); + + var init = tf.global_variables_initializer(); + var sess = tf.Session(); + sess.run(init); + + var (boxes, scores, classes, valid_detections) = tf.image.combined_non_max_suppression(boxes1, scores1, 10, 10, 0.5f, 0.2f, clip_boxes: false); + var result = sess.run((boxes, scores, classes, valid_detections)); var boxes_gt = tf.constant(new float[,] { { 210f, 112f, 150f, 100f }, { 200f, 100f, 150f, 100f }, { 190f, 110f, 150f, 100f }, { 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f, 0f},{ 0f, 0f, 0f , 0f},{ 0f, 0f, 0f, 0f},{ 0f , 0f, 0f, 0f},{ 0f, 0f, 0f, 0f} }); - boxes_gt = tf.reshape(boxes_gt,(1, 10, 4)); - Assert.AreEqual(boxes.numpy(), boxes_gt.numpy()); + boxes_gt = tf.reshape(boxes_gt, (1, 10, 4)); + Assert.AreEqual(result.Item1.numpy(), boxes_gt.numpy()); var scores_gt = tf.constant(new float[,] { { 0.9f, 0.7f, 0.3f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); scores_gt = tf.reshape(scores_gt, (1, 10)); - Assert.AreEqual(scores.numpy(), scores_gt.numpy()); + Assert.AreEqual(result.Item2.numpy(), scores_gt.numpy()); var classes_gt = tf.constant(new float[,] { { 1f, 1f, 0f, 0f, 0f, 0f, 0f, 0f, 0f, 0f } }); classes_gt = tf.reshape(classes_gt, (1, 10)); - Assert.AreEqual(classes.numpy(), classes_gt.numpy()); + Assert.AreEqual(result.Item3.numpy(), classes_gt.numpy()); var valid_detections_gt = tf.constant(new int[,] { { 3 } }); valid_detections_gt = tf.reshape(valid_detections_gt, (1)); - Assert.AreEqual(valid_detections.numpy(), valid_detections_gt.numpy()); + Assert.AreEqual(result.Item4.numpy(), valid_detections_gt.numpy()); } [TestMethod] From 005476cbcd71f4bcdfeda8f41461ea20dbdc09df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Wed, 26 Jul 2023 15:31:06 +0800 Subject: [PATCH 70/77] fix: add the gradient of the tf.gradient opr --- src/TensorFlowNET.Core/Gradients/array_grad.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/TensorFlowNET.Core/Gradients/array_grad.cs b/src/TensorFlowNET.Core/Gradients/array_grad.cs index 1b6bc95e..4b702799 100644 --- a/src/TensorFlowNET.Core/Gradients/array_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/array_grad.cs @@ -373,5 +373,13 @@ namespace Tensorflow.Gradients var p = op.inputs[1]; return new Tensor[] { array_ops.transpose(grads[0], array_ops.invert_permutation(p)), null }; } + + [RegisterGradient("ReverseV2")] + public static Tensor[] _ReverseV2Grad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var axis = op.inputs[1]; + return new Tensor[] { array_ops.reverse(grad, axis), null }; + } } } From f3b3d8be65f8d037dd456d6380bb93d2e888b53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <“583087864@qq.com”> Date: Fri, 28 Jul 2023 12:42:11 +0800 Subject: [PATCH 71/77] fix: add the momentum parameter's implemention of SGD --- src/TensorFlowNET.Core/Keras/IOptimizerApi.cs | 2 +- .../Training/gen_training_ops.cs | 4 ++++ .../Optimizers/OptimizerApi.cs | 4 ++-- src/TensorFlowNET.Keras/Optimizers/SGD.cs | 19 ++++++++++++++++++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs index d0d3a74f..19e3a7b8 100644 --- a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs +++ b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs @@ -63,6 +63,6 @@ namespace Tensorflow.Keras bool centered = false, string name = "RMSprop"); - IOptimizer SGD(float learning_rate); + IOptimizer SGD(float learning_rate, float momentum); } } diff --git a/src/TensorFlowNET.Core/Training/gen_training_ops.cs b/src/TensorFlowNET.Core/Training/gen_training_ops.cs index abe85a14..df7dd9e6 100644 --- a/src/TensorFlowNET.Core/Training/gen_training_ops.cs +++ b/src/TensorFlowNET.Core/Training/gen_training_ops.cs @@ -51,5 +51,9 @@ namespace Tensorflow public static Tensor resource_apply_gradient_descent(Tensor var, Tensor alpha, Tensor delta, bool use_locking = false, string name = null) => tf.Context.ExecuteOp("ResourceApplyGradientDescent", name, new ExecuteOpArgs(var, alpha, delta).SetAttributes(new { use_locking })); + + public static Tensor resource_apply_keras_momentum(Tensor var, Tensor accum, Tensor lr, Tensor grad, Tensor momentum, bool use_locking = false, bool use_nesterov = false, string name = null) + => tf.Context.ExecuteOp("ResourceApplyKerasMomentum", name, + new ExecuteOpArgs(var, accum, lr, grad, momentum).SetAttributes(new { use_locking, use_nesterov })); } } diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs index 28069426..affd43a4 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs @@ -71,7 +71,7 @@ namespace Tensorflow.Keras.Optimizers Name = name }); - public IOptimizer SGD(float learning_rate) - => new SGD(learning_rate); + public IOptimizer SGD(float learning_rate, float momentum) + => new SGD(learning_rate, momentum); } } diff --git a/src/TensorFlowNET.Keras/Optimizers/SGD.cs b/src/TensorFlowNET.Keras/Optimizers/SGD.cs index f97f4b15..1d9ceb81 100644 --- a/src/TensorFlowNET.Keras/Optimizers/SGD.cs +++ b/src/TensorFlowNET.Keras/Optimizers/SGD.cs @@ -22,6 +22,8 @@ namespace Tensorflow.Keras.Optimizers _set_hyper("decay", decay); _momentum = momentum > 0; + if (momentum < 0 || momentum > 1) + throw new ValueError($"momentum must be a number between 0 and 1, got {momentum}."); _set_hyper("momentum", momentum); @@ -30,6 +32,13 @@ namespace Tensorflow.Keras.Optimizers #pragma warning restore CS1717 // Assignment made to same variable } + protected override void _create_slots(IVariableV1[] var_list) + { + if (_momentum) + foreach (var var in var_list) + add_slot(var, "momentum"); + } + protected override void _prepare_local(DeviceDType device_dtype, Dictionary> _apply_state) { @@ -43,7 +52,15 @@ namespace Tensorflow.Keras.Optimizers { if (_momentum) { - throw new NotImplementedException("_resource_apply_dense"); + var momentum_var = get_slot(var, "momentum"); + return gen_training_ops.resource_apply_keras_momentum( + var.Handle, + momentum_var.Handle, + _get_hyper("learning_rate", var.dtype), + grad, + _get_hyper("momentum", var.dtype), + use_locking: _use_locking, + use_nesterov: nesterov); } var device_dtype = _apply_state.Keys.FirstOrDefault(x => x.Device == var.Device && x.DType == var.dtype.as_base_dtype()); From 6d3f134637308c4a4f01f49ca9e3b0222644a87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CWanglongzhi2001=E2=80=9D?= <583087864@qq.com> Date: Sat, 29 Jul 2023 15:48:13 +0800 Subject: [PATCH 72/77] fix: remove the reflection in the implemention of Bidirectional --- .../Layers/Rnn/Bidirectional.cs | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs index 6114d9c7..0566b08a 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/Bidirectional.cs @@ -13,17 +13,17 @@ namespace Tensorflow.Keras.Layers /// public class Bidirectional: Wrapper { - BidirectionalArgs _args; - RNN _forward_layer; - RNN _backward_layer; - RNN _layer; - bool _support_masking = true; int _num_constants = 0; + bool _support_masking = true; bool _return_state; bool _stateful; bool _return_sequences; - InputSpec _input_spec; + BidirectionalArgs _args; RNNArgs _layer_args_copy; + RNN _forward_layer; + RNN _backward_layer; + RNN _layer; + InputSpec _input_spec; public Bidirectional(BidirectionalArgs args):base(args) { _args = args; @@ -66,12 +66,16 @@ namespace Tensorflow.Keras.Layers // Recreate the forward layer from the original layer config, so that it // will not carry over any state from the layer. - var actualType = _layer.GetType(); - if (actualType == typeof(LSTM)) + if (_layer is LSTM) { var arg = _layer_args_copy as LSTMArgs; _forward_layer = new LSTM(arg); } + else if(_layer is SimpleRNN) + { + var arg = _layer_args_copy as SimpleRNNArgs; + _forward_layer = new SimpleRNN(arg); + } // TODO(Wanglongzhi2001), add GRU if case. else { @@ -154,12 +158,18 @@ namespace Tensorflow.Keras.Layers { config.GoBackwards = !config.GoBackwards; } - var actualType = layer.GetType(); - if (actualType == typeof(LSTM)) + + if (layer is LSTM) { var arg = config as LSTMArgs; return new LSTM(arg); } + else if(layer is SimpleRNN) + { + var arg = config as SimpleRNNArgs; + return new SimpleRNN(arg); + } + // TODO(Wanglongzhi2001), add GRU if case. else { return new RNN(cell, config); @@ -183,7 +193,6 @@ namespace Tensorflow.Keras.Layers protected override Tensors Call(Tensors inputs, Tensors state = null, bool? training = null, IOptionalArgs? optional_args = null) { // `Bidirectional.call` implements the same API as the wrapped `RNN`. - Tensors forward_inputs; Tensors backward_inputs; Tensors forward_state; From f5eb4ff0a0950fa1b0c3af9b67950e4f4dc90a1a Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sat, 26 Aug 2023 10:35:45 +0800 Subject: [PATCH 73/77] fix: partially fix the bug of load_model --- .../ArgsDefinition/Activation/ExponentialArgs.cs | 10 ++++++++++ .../ArgsDefinition/Activation/HardSigmoidArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SELUArgs.cs | 11 +++++++++++ .../Keras/ArgsDefinition/Activation/SoftplusArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SoftsignArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/SwishArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Activation/TanhArgs.cs | 10 ++++++++++ .../ArgsDefinition/Convolution/Conv2DTransposeArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/AddArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/ConcatenateArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Merging/SubtractArgs.cs | 10 ++++++++++ .../Pooling/GlobalAveragePooling1DArgs.cs | 10 ++++++++++ .../Pooling/GlobalAveragePooling2DArgs.cs | 10 ++++++++++ .../ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs | 10 ++++++++++ .../ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs | 10 ++++++++++ .../Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs | 10 ++++++++++ 16 files changed, 161 insertions(+) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs new file mode 100644 index 00000000..ef024971 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ExponentialArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class ExponentialArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs new file mode 100644 index 00000000..788e0f36 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/HardSigmoidArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class HardSigmoidArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs new file mode 100644 index 00000000..eb0e1844 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SELUArgs.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SELUArgs : LayerArgs + { + + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs new file mode 100644 index 00000000..7b4f2079 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftplusArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SoftplusArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs new file mode 100644 index 00000000..4e23d261 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftsignArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SoftsignArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs new file mode 100644 index 00000000..3dea06a2 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SwishArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SwishArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs new file mode 100644 index 00000000..5df41b71 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/TanhArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class TanhArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs new file mode 100644 index 00000000..3daba946 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/Conv2DTransposeArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class Conv2DTransposeArgs : Conv2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs new file mode 100644 index 00000000..016d5820 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/AddArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class AddArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs new file mode 100644 index 00000000..4a81d139 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/ConcatenateArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class ConcatenateArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs new file mode 100644 index 00000000..1e3621cb --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/SubtractArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class SubtractArgs : MergeArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs new file mode 100644 index 00000000..e73aff76 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalAveragePooling1DArgs : Pooling1DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs new file mode 100644 index 00000000..d143cf47 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalAveragePooling2DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalAveragePooling2DArgs : Pooling2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs new file mode 100644 index 00000000..e03227fe --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalMaxPooling1DArgs : Pooling1DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs new file mode 100644 index 00000000..a95cac83 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/GlobalMaxPooling2DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GlobalMaxPooling2DArgs : Pooling2DArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs new file mode 100644 index 00000000..4cfff2c1 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/MaxPooling1DArgs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class MaxPooling1DArgs : Pooling1DArgs + { + } +} From f679af67e61c51bee1aca254f993d6d137df07ff Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Sat, 26 Aug 2023 11:36:41 +0800 Subject: [PATCH 74/77] fix: partially fix the bug of load_model --- .../Layers/LayersApi.Activation.cs | 14 +++++++------- .../Layers/LayersApi.Merging.cs | 2 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs index 280e91e2..2c55f8fd 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs @@ -10,14 +10,14 @@ namespace Tensorflow.Keras.Layers { public ILayer ELU ( float alpha = 0.1f ) => new ELU(new ELUArgs { Alpha = alpha }); public ILayer SELU () - => new SELU(new LayerArgs { }); + => new SELU(new SELUArgs { }); public ILayer Softmax(int axis = -1) => new Softmax(new SoftmaxArgs { axis = axis }); public ILayer Softmax ( Axis axis ) => new Softmax(new SoftmaxArgs { axis = axis }); - public ILayer Softplus () => new Softplus(new LayerArgs { }); - public ILayer HardSigmoid () => new HardSigmoid(new LayerArgs { }); - public ILayer Softsign () => new Softsign(new LayerArgs { }); - public ILayer Swish () => new Swish(new LayerArgs { }); - public ILayer Tanh () => new Tanh(new LayerArgs { }); - public ILayer Exponential () => new Exponential(new LayerArgs { }); + public ILayer Softplus () => new Softplus(new SoftplusArgs { }); + public ILayer HardSigmoid () => new HardSigmoid(new HardSigmoidArgs { }); + public ILayer Softsign () => new Softsign(new SoftsignArgs { }); + public ILayer Swish () => new Swish(new SwishArgs { }); + public ILayer Tanh () => new Tanh(new TanhArgs { }); + public ILayer Exponential () => new Exponential(new ExponentialArgs { }); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs index d94bfb4d..bf06b141 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs @@ -14,7 +14,7 @@ namespace Tensorflow.Keras.Layers /// Axis along which to concatenate. /// public ILayer Concatenate(int axis = -1) - => new Concatenate(new MergeArgs + => new Concatenate(new ConcatenateArgs { Axis = axis }); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index a04a9c05..9155c774 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -240,7 +240,7 @@ namespace Tensorflow.Keras.Layers string kernel_regularizer = null, string bias_regularizer = null, string activity_regularizer = null) - => new Conv2DTranspose(new Conv2DArgs + => new Conv2DTranspose(new Conv2DTransposeArgs { Rank = 2, Filters = filters, @@ -568,7 +568,7 @@ namespace Tensorflow.Keras.Layers int? strides = null, string padding = "valid", string data_format = null) - => new MaxPooling1D(new Pooling1DArgs + => new MaxPooling1D(new MaxPooling1DArgs { PoolSize = pool_size ?? 2, Strides = strides ?? (pool_size ?? 2), @@ -944,21 +944,21 @@ namespace Tensorflow.Keras.Layers /// /// public ILayer Add() - => new Add(new MergeArgs { }); + => new Add(new AddArgs { }); /// /// /// /// public ILayer Subtract() - => new Subtract(new MergeArgs { }); + => new Subtract(new SubtractArgs { }); /// /// Global max pooling operation for spatial data. /// /// public ILayer GlobalAveragePooling2D() - => new GlobalAveragePooling2D(new Pooling2DArgs { }); + => new GlobalAveragePooling2D(new GlobalAveragePooling2DArgs { }); /// /// Global average pooling operation for temporal data. @@ -968,7 +968,7 @@ namespace Tensorflow.Keras.Layers /// /// public ILayer GlobalAveragePooling1D(string data_format = "channels_last") - => new GlobalAveragePooling1D(new Pooling1DArgs { DataFormat = data_format }); + => new GlobalAveragePooling1D(new GlobalAveragePooling1DArgs { DataFormat = data_format }); /// /// Global max pooling operation for spatial data. @@ -977,7 +977,7 @@ namespace Tensorflow.Keras.Layers /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// public ILayer GlobalAveragePooling2D(string data_format = "channels_last") - => new GlobalAveragePooling2D(new Pooling2DArgs { DataFormat = data_format }); + => new GlobalAveragePooling2D(new GlobalAveragePooling2DArgs { DataFormat = data_format }); /// /// Global max pooling operation for 1D temporal data. @@ -988,7 +988,7 @@ namespace Tensorflow.Keras.Layers /// /// public ILayer GlobalMaxPooling1D(string data_format = "channels_last") - => new GlobalMaxPooling1D(new Pooling1DArgs { DataFormat = data_format }); + => new GlobalMaxPooling1D(new GlobalMaxPooling1DArgs { DataFormat = data_format }); /// /// Global max pooling operation for spatial data. @@ -997,7 +997,7 @@ namespace Tensorflow.Keras.Layers /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// public ILayer GlobalMaxPooling2D(string data_format = "channels_last") - => new GlobalMaxPooling2D(new Pooling2DArgs { DataFormat = data_format }); + => new GlobalMaxPooling2D(new GlobalMaxPooling2DArgs { DataFormat = data_format }); /// /// Get an weights initializer from its name. From 8e3ba22c832e6d34598644686e00182924b08c3a Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Sat, 26 Aug 2023 16:29:28 +0800 Subject: [PATCH 75/77] fix: validate dataset of `Imdb` do not load bug & add: custom `Imdb` path --- src/TensorFlowNET.Keras/Datasets/Imdb.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Keras/Datasets/Imdb.cs b/src/TensorFlowNET.Keras/Datasets/Imdb.cs index 61ce3947..a62f3f87 100644 --- a/src/TensorFlowNET.Keras/Datasets/Imdb.cs +++ b/src/TensorFlowNET.Keras/Datasets/Imdb.cs @@ -31,7 +31,7 @@ namespace Tensorflow.Keras.Datasets /// /// /// - public DatasetPass load_data(string path = "imdb.npz", + public DatasetPass load_data(string? path = "imdb.npz", int num_words = -1, int skip_top = 0, int maxlen = -1, @@ -42,7 +42,7 @@ namespace Tensorflow.Keras.Datasets { if (maxlen == -1) throw new InvalidArgumentError("maxlen must be assigned."); - var dst = Download(); + var dst = path ?? Download(); var lines = File.ReadAllLines(Path.Combine(dst, "imdb_train.txt")); var x_train_string = new string[lines.Length]; @@ -55,7 +55,7 @@ namespace Tensorflow.Keras.Datasets var x_train = keras.preprocessing.sequence.pad_sequences(PraseData(x_train_string), maxlen: maxlen); - File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); + lines = File.ReadAllLines(Path.Combine(dst, "imdb_test.txt")); var x_test_string = new string[lines.Length]; var y_test = np.zeros(new int[] { lines.Length }, np.int64); for (int i = 0; i < lines.Length; i++) From ba1ddb44488bbb2f528065ac2be07e9e6965722e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 26 Aug 2023 11:20:12 -0500 Subject: [PATCH 76/77] Set SGD default value. --- src/TensorFlowNET.Core/Keras/IOptimizerApi.cs | 2 +- .../Tensorflow.Binding.csproj | 10 ++--- .../Optimizers/OptimizerApi.cs | 2 +- .../Tensorflow.Keras.csproj | 39 ++++++++++--------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs index 19e3a7b8..6c15fd46 100644 --- a/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs +++ b/src/TensorFlowNET.Core/Keras/IOptimizerApi.cs @@ -63,6 +63,6 @@ namespace Tensorflow.Keras bool centered = false, string name = "RMSprop"); - IOptimizer SGD(float learning_rate, float momentum); + IOptimizer SGD(float learning_rate = 0.01f, float momentum = 0f); } } diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index ca5aa47a..babb5256 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,13 +5,13 @@ Tensorflow.Binding Tensorflow 2.11.0 - 0.110.2 + 0.110.3 10.0 enable Haiping Chen, Eli Belash, Yaohui Liu, Meinrad Recheis SciSharp STACK False - Apache 2.0, Haiping Chen $([System.DateTime]::UtcNow.ToString(yyyy)) + Apache 2.0, Haiping Chen since 2018 https://github.com/SciSharp/TensorFlow.NET git http://scisharpstack.org @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.110.1.0 + 0.110.3.0 tf.net 0.110.x and above are based on tensorflow native 2.11.0 * Support RNN, LSTM model. @@ -43,7 +43,7 @@ https://tensorflownet.readthedocs.io tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. tf.net 0.11x.x aligns with TensorFlow v2.11.x native library. - 0.110.2.0 + 0.110.3.0 LICENSE true packages @@ -172,7 +172,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs index affd43a4..a237499f 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerApi.cs @@ -71,7 +71,7 @@ namespace Tensorflow.Keras.Optimizers Name = name }); - public IOptimizer SGD(float learning_rate, float momentum) + public IOptimizer SGD(float learning_rate = 0.01f, float momentum = 0f) => new SGD(learning_rate, momentum); } } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index eeb7c559..36d1bc1d 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,27 +7,30 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.11.2 + 0.11.3 Haiping Chen Keras for .NET - Apache 2.0, Haiping Chen 2023 + Apache 2.0, Haiping Chen since 2018 TensorFlow.Keras https://github.com/SciSharp/TensorFlow.NET https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 https://github.com/SciSharp/TensorFlow.NET - Keras for .NET is a C# version of Keras ported from the python version. - -* Support CIFAR-10 dataset in keras.datasets. -* Support Conv2D functional API. -* Support BatchNormalization layer. -* Building keras model in subclass, functional and sequential api -* Implemented backward_function. -* Support model.load_weights. -* Add Subtract layer -* Text preprocessing -* Preprocessing.timeseries_dataset_from_array -* Fixed memory leak for YOLOv3 model. -* Support RNN and LSTM models + + Keras for .NET is a C# version of Keras ported from the python version. + + * Support CIFAR-10 dataset in keras.datasets. + * Support Conv2D functional API. + * Support BatchNormalization layer. + * Building keras model in subclass, functional and sequential api + * Implemented backward_function. + * Support model.load_weights. + * Add Subtract layer + * Text preprocessing + * Preprocessing.timeseries_dataset_from_array + * Fixed memory leak for YOLOv3 model. + * Support RNN and LSTM models + * Support Transformer model + Keras for .NET Keras is an API designed for human beings, not machines. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear & actionable error messages. @@ -39,8 +42,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git False Open.snk - 0.11.2.0 - 0.11.2.0 + 0.11.3.0 + 0.11.3.0 LICENSE Debug;Release;GPU @@ -140,7 +143,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + From 7b077eac7e6a9e60d9d34be9782e222317fbe353 Mon Sep 17 00:00:00 2001 From: Wanglongzhi2001 <583087864@qq.com> Date: Mon, 4 Sep 2023 00:05:22 +0800 Subject: [PATCH 77/77] feat: implement GRU layer --- .../Keras/ArgsDefinition/Rnn/GRUArgs.cs | 29 +++ .../ArgsDefinition/Rnn/GRUOptionalArgs.cs | 13 ++ .../Keras/Layers/ILayersApi.cs | 19 ++ src/TensorFlowNET.Keras/Layers/LayersApi.cs | 61 ++++++- src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs | 168 ++++++++++++++++++ src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 42 +---- .../Layers/Rnn.Test.cs | 9 + 7 files changed, 300 insertions(+), 41 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs new file mode 100644 index 00000000..cdc3097e --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUArgs.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GRUArgs : AutoSerializeLayerArgs + { + public int Units { get; set; } + public Activation Activation { get; set; } + public Activation RecurrentActivation { get; set; } + public bool UseBias { get; set; } = true; + public float Dropout { get; set; } = .0f; + public float RecurrentDropout { get; set; } = .0f; + public IInitializer KernelInitializer { get; set; } + public IInitializer RecurrentInitializer { get; set; } + public IInitializer BiasInitializer { get; set; } + public bool ReturnSequences { get;set; } + public bool ReturnState { get;set; } + public bool GoBackwards { get;set; } + public bool Stateful { get;set; } + public bool Unroll { get;set; } + public bool TimeMajor { get;set; } + public bool ResetAfter { get;set; } + public int Implementation { get; set; } = 2; + + } + +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs new file mode 100644 index 00000000..d441dc82 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/GRUOptionalArgs.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class GRUOptionalArgs + { + public string Identifier => "GRU"; + + public Tensor Mask { get; set; } = null; + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index b8aff5fb..5e08eadc 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -259,6 +259,25 @@ namespace Tensorflow.Keras.Layers float recurrent_dropout = 0f, bool reset_after = true); + public ILayer GRU( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false, + bool reset_after = true + ); + /// /// Bidirectional wrapper for RNNs. /// diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 9155c774..928e7e33 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -784,7 +784,7 @@ namespace Tensorflow.Keras.Layers string recurrent_activation = "sigmoid", bool use_bias = true, string kernel_initializer = "glorot_uniform", - string recurrent_initializer = "orthogonal", // TODO(Wanglongzhi2001),glorot_uniform has not been developed. + string recurrent_initializer = "orthogonal", string bias_initializer = "zeros", bool unit_forget_bias = true, float dropout = 0f, @@ -908,6 +908,65 @@ namespace Tensorflow.Keras.Layers ResetAfter = reset_after }); + /// + /// Gated Recurrent Unit - Cho et al. 2014. + /// + /// Positive integer, dimensionality of the output space. + /// Activation function to use. If you pass `None`, no activation is applied.(ie. "linear" activation: `a(x) = x`). + /// Activation function to use for the recurrent step. If you pass `None`, no activation is applied. (ie. "linear" activation: `a(x) = x`). + /// Boolean, (default `True`), whether the layer uses a bias vector. + /// Initializer for the `kernel` weights matrix, used for the linear transformation of the inputs. Default: `glorot_uniform`. + /// Initializer for the `recurrent_kernel` weights matrix, used for the linear transformation of the recurrent state. Default: `orthogonal`. + /// Initializer for the bias vector. Default: `zeros`. + /// Float between 0 and 1. Fraction of the units to drop for the linear transformation of the inputs. Default: 0. + /// Float between 0 and 1. Fraction of the units to drop for the linear transformation of the recurrent state. Default: 0. + /// + /// Boolean. Whether to return the last output in the output sequence, or the full sequence. Default: `False`. + /// Boolean. Whether to return the last state in addition to the output. Default: `False`. + /// Boolean (default `False`). If True, process the input sequence backwards and return the reversed sequence. + /// Boolean (default False). If True, the last state for each sample at index i in a batch will be used as initial state for the sample of index i in the following batch. + /// Boolean (default False). If True, the network will be unrolled, else a symbolic loop will be used. Unrolling can speed-up a RNN, + /// The shape format of the `inputs` and `outputs` tensors. + /// GRU convention (whether to apply reset gate after or before matrix multiplication). False = "before", True = "after" (default and cuDNN compatible). + /// + public ILayer GRU( + int units, + string activation = "tanh", + string recurrent_activation = "sigmoid", + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros", + float dropout = 0f, + float recurrent_dropout = 0f, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool unroll = false, + bool time_major = false, + bool reset_after = true + ) + => new GRU(new GRUArgs + { + Units = units, + Activation = keras.activations.GetActivationFromName(activation), + RecurrentActivation = keras.activations.GetActivationFromName(recurrent_activation), + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + UseBias = use_bias, + Dropout = dropout, + RecurrentDropout = recurrent_dropout, + ReturnSequences = return_sequences, + ReturnState = return_state, + GoBackwards = go_backwards, + Stateful = stateful, + TimeMajor = time_major, + Unroll = unroll, + ResetAfter = reset_after + }); + public ILayer Bidirectional( ILayer layer, string merge_mode = "concat", diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs b/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs new file mode 100644 index 00000000..0919883d --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/GRU.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Common.Extensions; +using Tensorflow.Common.Types; +using Tensorflow.Keras.Saving; + + +namespace Tensorflow.Keras.Layers +{ + public class GRU : RNN + { + GRUArgs _args; + private static GRUCell _cell; + + bool _return_runtime; + public GRUCell Cell { get => _cell; } + public int units { get => _args.Units; } + public Activation activation { get => _args.Activation; } + public Activation recurrent_activation { get => _args.RecurrentActivation; } + public bool use_bias { get => _args.UseBias; } + public float dropout { get => _args.Dropout; } + public float recurrent_dropout { get => _args.RecurrentDropout; } + public IInitializer kernel_initializer { get => _args.KernelInitializer; } + public IInitializer recurrent_initializer { get => _args.RecurrentInitializer; } + public IInitializer bias_initializer { get => _args.BiasInitializer; } + public int implementation { get => _args.Implementation; } + public bool reset_after { get => _args.ResetAfter; } + + public GRU(GRUArgs args) : base(CreateCell(args), PreConstruct(args)) + { + _args = args; + + if (_args.Implementation == 0) + { + // Use the red output to act as a warning message that can also be used under the release version + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Warning: `implementation=0` has been deprecated, "+ + "and now defaults to `implementation=2`."+ + "Please update your layer call."); + Console.ResetColor(); + } + + GRUCell cell = new GRUCell(new GRUCellArgs + { + Units = _args.Units, + Activation = _args.Activation, + RecurrentActivation = _args.RecurrentActivation, + UseBias = _args.UseBias, + Dropout = _args.Dropout, + RecurrentDropout = _args.RecurrentDropout, + KernelInitializer = _args.KernelInitializer, + RecurrentInitializer = _args.RecurrentInitializer, + BiasInitializer = _args.BiasInitializer, + ResetAfter = _args.ResetAfter, + Implementation = _args.Implementation + }); + _cell = cell; + } + + protected override Tensors Call(Tensors inputs, Tensors initial_state = null, bool? training = null, IOptionalArgs? optional_args = null) + { + GRUOptionalArgs? gru_optional_args = optional_args as GRUOptionalArgs; + if (optional_args is not null && gru_optional_args is null) + { + throw new ArgumentException("The type of optional args should be `GRUOptionalArgs`."); + } + Tensors? mask = gru_optional_args?.Mask; + + // Not support ragger input temporarily; + int row_length = 0; + bool is_ragged_input = false; + + _validate_args_if_ragged(is_ragged_input, mask); + + // GRU does not support constants.Ignore it during process. + (inputs, initial_state, _) = this._process_inputs(inputs, initial_state, null); + + if (mask.Length > 1) + { + mask = mask[0]; + } + + var input_shape = inputs.shape; + var timesteps = _args.TimeMajor ? input_shape[0] : input_shape[1]; + + + // TODO(Wanglongzhi2001), finish _could_use_gpu_kernel part + Func step = (cell_inputs, cell_states) => + { + var res = Cell.Apply(cell_inputs, cell_states, training is null ? true : training.Value); + var (output, state) = res; + return (output, state); + }; + + var (last_output, outputs, states) = keras.backend.rnn( + step, + inputs, + initial_state, + constants: null, + go_backwards: _args.GoBackwards, + mask: mask, + unroll: _args.Unroll, + input_length: ops.convert_to_tensor(timesteps), + time_major: _args.TimeMajor, + zero_output_for_mask: base.Args.ZeroOutputForMask, + return_all_outputs: _args.ReturnSequences + ); + + Tensors output; + if (_args.ReturnSequences) + { + output = outputs; + } + else + { + output = last_output; + } + + if (_args.ReturnState) + { + output = new Tensors { output, states }; + } + return output; + } + + private static IRnnCell CreateCell(GRUArgs gruArgs) + { + return new GRUCell(new GRUCellArgs + { + Units = gruArgs.Units, + Activation = gruArgs.Activation, + RecurrentActivation = gruArgs.RecurrentActivation, + UseBias = gruArgs.UseBias, + Dropout = gruArgs.Dropout, + RecurrentDropout = gruArgs.RecurrentDropout, + KernelInitializer = gruArgs.KernelInitializer, + RecurrentInitializer = gruArgs.RecurrentInitializer, + BiasInitializer = gruArgs.BiasInitializer, + ResetAfter = gruArgs.ResetAfter, + Implementation = gruArgs.Implementation + }); + } + + private static RNNArgs PreConstruct(GRUArgs args) + { + return new RNNArgs + { + ReturnSequences = args.ReturnSequences, + ReturnState = args.ReturnState, + GoBackwards = args.GoBackwards, + Stateful = args.Stateful, + Unroll = args.Unroll, + TimeMajor = args.TimeMajor, + Units = args.Units, + Activation = args.Activation, + RecurrentActivation = args.RecurrentActivation, + UseBias = args.UseBias, + Dropout = args.Dropout, + RecurrentDropout = args.RecurrentDropout, + KernelInitializer = args.KernelInitializer, + RecurrentInitializer = args.RecurrentInitializer, + BiasInitializer = args.BiasInitializer + }; + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index c1922261..fec75559 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -25,8 +25,8 @@ namespace Tensorflow.Keras.Layers private RNNArgs _args; private object _input_spec = null; // or NoneValue?? private object _state_spec = null; - private Tensors _states = null; private object _constants_spec = null; + private Tensors _states = null; private int _num_constants; protected IVariableV1 _kernel; protected IVariableV1 _bias; @@ -469,7 +469,7 @@ namespace Tensorflow.Keras.Layers return (inputs, initial_state, constants); } - private void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) + protected void _validate_args_if_ragged(bool is_ragged_input, Tensors mask) { if (!is_ragged_input) { @@ -528,44 +528,6 @@ namespace Tensorflow.Keras.Layers throw new NotImplementedException(); } - // 好像不能cell不能传接口类型 - //public RNN New(IRnnArgCell cell, - // bool return_sequences = false, - // bool return_state = false, - // bool go_backwards = false, - // bool stateful = false, - // bool unroll = false, - // bool time_major = false) - // => new RNN(new RNNArgs - // { - // Cell = cell, - // ReturnSequences = return_sequences, - // ReturnState = return_state, - // GoBackwards = go_backwards, - // Stateful = stateful, - // Unroll = unroll, - // TimeMajor = time_major - // }); - - //public RNN New(List cell, - // bool return_sequences = false, - // bool return_state = false, - // bool go_backwards = false, - // bool stateful = false, - // bool unroll = false, - // bool time_major = false) - // => new RNN(new RNNArgs - // { - // Cell = cell, - // ReturnSequences = return_sequences, - // ReturnState = return_state, - // GoBackwards = go_backwards, - // Stateful = stateful, - // Unroll = unroll, - // TimeMajor = time_major - // }); - - protected Tensors get_initial_state(Tensors inputs) { var input = inputs[0]; diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs index 03159346..dbf5cae1 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/Rnn.Test.cs @@ -146,6 +146,15 @@ namespace Tensorflow.Keras.UnitTest.Layers } + [TestMethod] + public void GRU() + { + var inputs = tf.ones((32, 10, 8)); + var gru = tf.keras.layers.GRU(4); + var output = gru.Apply(inputs); + Assert.AreEqual((32, 4), output.shape); + } + [TestMethod] public void Bidirectional() {