From e64bc45d713447f75661256911a75aadc7721641 Mon Sep 17 00:00:00 2001 From: lingbai-kong Date: Sun, 21 May 2023 16:22:04 +0800 Subject: [PATCH] Replace old dense layer with a new that has regularizer and constraint params. --- .../Keras/Layers/ILayersApi.cs | 11 +-- src/TensorFlowNET.Keras/Layers/Core/Dense.cs | 4 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 81 +++++++++---------- 3 files changed, 47 insertions(+), 49 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 6a29f9e5..ddb44afc 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -94,16 +94,17 @@ namespace Tensorflow.Keras.Layers string kernel_initializer = "glorot_uniform", string bias_initializer = "zeros"); - public ILayer Dense(int units); - public ILayer Dense(int units, - string activation = null, - Shape input_shape = null); public ILayer Dense(int units, Activation activation = null, IInitializer kernel_initializer = null, bool use_bias = true, IInitializer bias_initializer = null, - Shape input_shape = null); + Shape input_shape = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null, + Action kernel_constraint = null, + Action bias_constraint = null); public ILayer Dropout(float rate, Shape noise_shape = null, int? seed = null); diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index aa6617dd..5806d1af 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -56,6 +56,8 @@ namespace Tensorflow.Keras.Layers "kernel", shape: new Shape(last_dim, args.Units), initializer: args.KernelInitializer, + regularizer: args.KernelRegularizer, + // constraint: args.KernelConstraint, dtype: DType, trainable: true); if (args.UseBias) @@ -63,6 +65,8 @@ namespace Tensorflow.Keras.Layers "bias", shape: new Shape(args.Units), initializer: args.BiasInitializer, + regularizer: args.BiasRegularizer, + // constraint: args.BiasConstraint, dtype: DType, trainable: true); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 3b095bc2..2379c7af 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -262,7 +262,7 @@ namespace Tensorflow.Keras.Layers /// /// Just your regular densely-connected NN layer. /// - /// Dense implements the operation: output = activation(dot(input, kernel) + bias) where activation is the + /// Dense implements the operation: output = activation(input * kernel + bias) where activation is the /// element-wise activation function passed as the activation argument, kernel is a weights matrix created by the layer, /// and bias is a bias vector created by the layer (only applicable if use_bias is True). /// @@ -272,57 +272,35 @@ namespace Tensorflow.Keras.Layers /// Boolean, whether the layer uses a bias vector. /// Initializer for the bias vector. /// N-D tensor with shape: (batch_size, ..., input_dim). The most common situation would be a 2D input with shape (batch_size, input_dim). + /// Regularizer instance for the kernel matrix (callable). + /// Regularizer instance for the bias (callable). + /// Regularizer instance for the output (callable). + /// Constraint function for the kernel matrix. + /// Constraint function for the bias. /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). public ILayer Dense(int units, Activation activation = null, IInitializer kernel_initializer = null, bool use_bias = true, IInitializer bias_initializer = null, - Shape input_shape = null) + Shape input_shape = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null, + Action kernel_constraint = null, + Action bias_constraint = null) => new Dense(new DenseArgs { Units = units, Activation = activation ?? keras.activations.Linear, KernelInitializer = kernel_initializer ?? tf.glorot_uniform_initializer, BiasInitializer = bias_initializer ?? (use_bias ? tf.zeros_initializer : null), - InputShape = input_shape - }); - - /// - /// Just your regular densely-connected NN layer. - /// - /// Dense implements the operation: output = activation(dot(input, kernel) + bias) where activation is the - /// element-wise activation function passed as the activation argument, kernel is a weights matrix created by the layer, - /// and bias is a bias vector created by the layer (only applicable if use_bias is True). - /// - /// Positive integer, dimensionality of the output space. - /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). - public ILayer Dense(int units) - => new Dense(new DenseArgs - { - Units = units, - Activation = keras.activations.GetActivationFromName("linear") - }); - - /// - /// Just your regular densely-connected NN layer. - /// - /// Dense implements the operation: output = activation(dot(input, kernel) + bias) where activation is the - /// element-wise activation function passed as the activation argument, kernel is a weights matrix created by the layer, - /// and bias is a bias vector created by the layer (only applicable if use_bias is True). - /// - /// Positive integer, dimensionality of the output space. - /// Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x). - /// N-D tensor with shape: (batch_size, ..., input_dim). The most common situation would be a 2D input with shape (batch_size, input_dim). - /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). - public ILayer Dense(int units, - string activation = null, - Shape input_shape = null) - => new Dense(new DenseArgs - { - Units = units, - Activation = keras.activations.GetActivationFromName(activation), - InputShape = input_shape + InputShape = input_shape, + KernelRegularizer = kernel_regularizer, + BiasRegularizer = bias_regularizer, + ActivityRegularizer = activity_regularizer, + KernelConstraint = kernel_constraint, + BiasConstraint = bias_constraint }); /// @@ -331,10 +309,15 @@ namespace Tensorflow.Keras.Layers /// /// /// Python integer, dimensionality of the output space. - /// - /// Boolean, whether the layer uses a bias. - /// - /// + /// Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x). + /// Initializer for the kernel weights matrix. + /// Boolean, whether the layer uses a bias vector. + /// Initializer for the bias vector. + /// Regularizer instance for the kernel matrix (callable). + /// Regularizer instance for the bias (callable). + /// Regularizer instance for the output (callable). + /// Constraint function for the kernel matrix. + /// Constraint function for the bias. /// /// /// @@ -345,6 +328,11 @@ namespace Tensorflow.Keras.Layers bool use_bias = true, IInitializer kernel_initializer = null, IInitializer bias_initializer = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null, + Action kernel_constraint = null, + Action bias_constraint = null, bool trainable = true, string name = null, bool? reuse = null) @@ -359,6 +347,11 @@ namespace Tensorflow.Keras.Layers UseBias = use_bias, BiasInitializer = bias_initializer, KernelInitializer = kernel_initializer, + KernelRegularizer = kernel_regularizer, + BiasRegularizer = bias_regularizer, + ActivityRegularizer = activity_regularizer, + KernelConstraint = kernel_constraint, + BiasConstraint = bias_constraint, Trainable = trainable, Name = name });