Browse Source

np.permutation

tags/v0.60-tf.numpy
Oceania2018 4 years ago
parent
commit
a4a4da9d30
19 changed files with 113 additions and 65 deletions
  1. +4
    -4
      src/TensorFlowNET.Core/APIs/tf.math.cs
  2. +2
    -0
      src/TensorFlowNET.Core/Binding.Util.cs
  3. +3
    -0
      src/TensorFlowNET.Core/Contexts/ExecuteOpArgs.cs
  4. +1
    -1
      src/TensorFlowNET.Core/NumPy/Axis.cs
  5. +29
    -0
      src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs
  6. +2
    -0
      src/TensorFlowNET.Core/NumPy/NDArray.Index.cs
  7. +8
    -2
      src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs
  8. +1
    -1
      src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs
  9. +3
    -0
      src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs
  10. +1
    -3
      src/TensorFlowNET.Core/Numpy/NDArray.cs
  11. +1
    -22
      src/TensorFlowNET.Core/Numpy/Numpy.cs
  12. +3
    -0
      src/TensorFlowNET.Core/Sessions/BaseSession.cs
  13. +1
    -1
      src/TensorFlowNET.Core/Tensors/constant_op.cs
  14. +18
    -23
      src/TensorFlowNET.Core/Tensors/tensor_util.cs
  15. +2
    -2
      src/TensorFlowNET.Core/Training/Saving/Saver.cs
  16. +2
    -2
      src/TensorFlowNET.Core/ops.cs
  17. +3
    -2
      src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.index_directory.cs
  18. +2
    -2
      src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs
  19. +27
    -0
      test/TensorFlowNET.UnitTest/NumPy/Randomize.Test.cs

+ 4
- 4
src/TensorFlowNET.Core/APIs/tf.math.cs View File

@@ -21,6 +21,9 @@ namespace Tensorflow
public MathApi math { get; } = new MathApi(); public MathApi math { get; } = new MathApi();
public class MathApi public class MathApi
{ {
public Tensor argmax(Tensor input, Axis axis = null, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
=> gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);

public Tensor log(Tensor x, string name = null) public Tensor log(Tensor x, string name = null)
=> gen_math_ops.log(x, name); => gen_math_ops.log(x, name);


@@ -539,15 +542,12 @@ namespace Tensorflow
public Tensor round(Tensor x, string name = null) public Tensor round(Tensor x, string name = null)
=> gen_math_ops.round(x, name: name); => gen_math_ops.round(x, name: name);


public Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null)
public Tensor cast(Tensor x, TF_DataType dtype, string name = null)
=> math_ops.cast(x, dtype, name); => math_ops.cast(x, dtype, name);


public Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null) public Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null)
=> math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name); => math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name);


public Tensor argmax(Tensor input, int axis = -1, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
=> gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);

public Tensor square(Tensor x, string name = null) public Tensor square(Tensor x, string name = null)
=> gen_math_ops.square(x, name: name); => gen_math_ops.square(x, name: name);
public Tensor squared_difference(Tensor x, Tensor y, string name = null) public Tensor squared_difference(Tensor x, Tensor y, string name = null)


+ 2
- 0
src/TensorFlowNET.Core/Binding.Util.cs View File

@@ -549,6 +549,8 @@ namespace Tensorflow
return tensors.dtype; return tensors.dtype;
case IEnumerable<Tensor> tensors: case IEnumerable<Tensor> tensors:
return tensors.First().dtype; return tensors.First().dtype;
case RefVariable variable:
return variable.dtype;
case ResourceVariable variable: case ResourceVariable variable:
return variable.dtype; return variable.dtype;
default: default:


+ 3
- 0
src/TensorFlowNET.Core/Contexts/ExecuteOpArgs.cs View File

@@ -1,5 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Text; using System.Text;
using static Tensorflow.Binding; using static Tensorflow.Binding;


@@ -11,11 +12,13 @@ namespace Tensorflow
public object[] OpInputArgs { get; set; } public object[] OpInputArgs { get; set; }
public Dictionary<string, object> OpAttrs { get; set; } public Dictionary<string, object> OpAttrs { get; set; }
[DebuggerStepThrough]
public ExecuteOpArgs(params object[] inputArgs) public ExecuteOpArgs(params object[] inputArgs)
{ {
OpInputArgs = inputArgs; OpInputArgs = inputArgs;
} }


[DebuggerStepThrough]
public ExecuteOpArgs SetAttributes(object attrs) public ExecuteOpArgs SetAttributes(object attrs)
{ {
OpAttrs = ConvertToDict(attrs); OpAttrs = ConvertToDict(attrs);


+ 1
- 1
src/TensorFlowNET.Core/NumPy/Axis.cs View File

@@ -56,7 +56,7 @@ namespace Tensorflow
=> constant_op.constant(axis); => constant_op.constant(axis);


public override string ToString() public override string ToString()
=> $"({string.Join(", ", axis)})";
=> IsScalar ? $"{axis[0]}" : $"({string.Join(", ", axis)})";
} }
} }




+ 29
- 0
src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Text;

namespace Tensorflow.NumPy
{
public class RandomizedImpl
{
[AutoNumPy]
public NDArray permutation(int x) => new NDArray(random_ops.random_shuffle(math_ops.range(0, x)));

[AutoNumPy]
public NDArray permutation(NDArray x) => new NDArray(random_ops.random_shuffle(x));

[AutoNumPy]
public void shuffle(NDArray x)
{
var y = random_ops.random_shuffle(x);
Marshal.Copy(y.BufferToArray(), 0, x.TensorDataPointer, (int)x.bytesize);
}

public NDArray rand(params int[] shape)
=> throw new NotImplementedException("");

public NDArray randint(long x)
=> throw new NotImplementedException("");
}
}

+ 2
- 0
src/TensorFlowNET.Core/NumPy/NDArray.Index.cs View File

@@ -45,6 +45,8 @@ namespace Tensorflow.NumPy
{ {
if(mask.dtype == TF_DataType.TF_INT32) if(mask.dtype == TF_DataType.TF_INT32)
return GetData(mask.ToArray<int>()); return GetData(mask.ToArray<int>());
else if (mask.dtype == TF_DataType.TF_INT64)
return GetData(mask.ToArray<long>().Select(x => Convert.ToInt32(x)).ToArray());


throw new NotImplementedException(""); throw new NotImplementedException("");
} }


+ 8
- 2
src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs View File

@@ -17,7 +17,13 @@ namespace Tensorflow.NumPy
=> new NDArray(math_ops.argmax(a, axis ?? -1)); => new NDArray(math_ops.argmax(a, axis ?? -1));


[AutoNumPy] [AutoNumPy]
public static NDArray unique(NDArray a)
=> throw new NotImplementedException("");
public static (NDArray, NDArray) unique(NDArray a)
{
var(u, indice) = array_ops.unique(a);
return (new NDArray(u), new NDArray(indice));
}

[AutoNumPy]
public static void shuffle(NDArray x) => np.random.shuffle(x);
} }
} }

+ 1
- 1
src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs View File

@@ -13,6 +13,6 @@ namespace Tensorflow.NumPy
public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.arg_min(x, axis)); public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.arg_min(x, axis));


[AutoNumPy] [AutoNumPy]
public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.arg_max(x, axis));
public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.math.argmax(x, axis));
} }
} }

+ 3
- 0
src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs View File

@@ -25,6 +25,9 @@ namespace Tensorflow.NumPy
public NDArray(byte[] bytes, Shape shape, TF_DataType dtype) public NDArray(byte[] bytes, Shape shape, TF_DataType dtype)
: base(bytes, shape, dtype) { NewEagerTensorHandle(); } : base(bytes, shape, dtype) { NewEagerTensorHandle(); }


public NDArray(long[] value, Shape? shape = null)
: base(value, shape) { NewEagerTensorHandle(); }

public NDArray(IntPtr address, Shape shape, TF_DataType dtype) public NDArray(IntPtr address, Shape shape, TF_DataType dtype)
: base(address, shape, dtype) { NewEagerTensorHandle(); } : base(address, shape, dtype) { NewEagerTensorHandle(); }




+ 1
- 3
src/TensorFlowNET.Core/Numpy/NDArray.cs View File

@@ -42,11 +42,9 @@ namespace Tensorflow.NumPy
public NDArray reshape(Shape newshape) => new NDArray(tf.reshape(this, newshape)); public NDArray reshape(Shape newshape) => new NDArray(tf.reshape(this, newshape));
public NDArray astype(TF_DataType dtype) => new NDArray(math_ops.cast(this, dtype)); public NDArray astype(TF_DataType dtype) => new NDArray(math_ops.cast(this, dtype));
public NDArray ravel() => throw new NotImplementedException(""); public NDArray ravel() => throw new NotImplementedException("");
public void shuffle(NDArray nd) => throw new NotImplementedException("");
public void shuffle(NDArray nd) => np.random.shuffle(nd);
public Array ToMuliDimArray<T>() => throw new NotImplementedException(""); public Array ToMuliDimArray<T>() => throw new NotImplementedException("");
public byte[] ToByteArray() => BufferToArray(); public byte[] ToByteArray() => BufferToArray();
public static string[] AsStringArray(NDArray arr) => throw new NotImplementedException("");

public override string ToString() => NDArrayRender.ToString(this); public override string ToString() => NDArrayRender.ToString(this);
} }
} }

+ 1
- 22
src/TensorFlowNET.Core/Numpy/Numpy.cs View File

@@ -75,28 +75,7 @@ namespace Tensorflow.NumPy
public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8, public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8,
bool equal_nan = false) => throw new NotImplementedException(""); bool equal_nan = false) => throw new NotImplementedException("");


public static class random
{
public static NDArray permutation(int x)
{
throw new NotImplementedException("");
}

public static void shuffle(NDArray nd)
{

}

public static NDArray rand(params int[] shape)
=> throw new NotImplementedException("");

public static NDArray randint(long x)
=> throw new NotImplementedException("");

public static NDArray RandomState(int x)
=> throw new NotImplementedException("");
}

public static RandomizedImpl random = new RandomizedImpl();
public static LinearAlgebraImpl linalg = new LinearAlgebraImpl(); public static LinearAlgebraImpl linalg = new LinearAlgebraImpl();
} }
} }

+ 3
- 0
src/TensorFlowNET.Core/Sessions/BaseSession.cs View File

@@ -193,6 +193,9 @@ namespace Tensorflow
case double v: case double v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v)); feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v));
break; break;
case string v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v));
break;
case Array v: case Array v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v, v.GetShape())); feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v, v.GetShape()));
break; break;


+ 1
- 1
src/TensorFlowNET.Core/Tensors/constant_op.cs View File

@@ -117,7 +117,7 @@ namespace Tensorflow
case Shape val: case Shape val:
return new EagerTensor(val.dims, new Shape(val.ndim)); return new EagerTensor(val.dims, new Shape(val.ndim));
case Axis val: case Axis val:
return new EagerTensor(val.axis, new Shape(val.size));
return new EagerTensor(val.axis, val.IsScalar ? Shape.Scalar : new Shape(val.size));
case string val: case string val:
return new EagerTensor(new[] { val }, Shape.Scalar); return new EagerTensor(new[] { val }, Shape.Scalar);
case string[] val: case string[] val:


+ 18
- 23
src/TensorFlowNET.Core/Tensors/tensor_util.cs View File

@@ -141,7 +141,24 @@ namespace Tensorflow
byte[] bytes = nd.ToByteArray(); byte[] bytes = nd.ToByteArray();
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes); tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
} }
else if (!values.GetType().IsArray)
else if (dtype == TF_DataType.TF_STRING && !(values is NDArray))
{
if (values is string str)
tensor_proto.StringVal.Add(Google.Protobuf.ByteString.CopyFromUtf8(str));
else if (values is string[] str_values)
tensor_proto.StringVal.AddRange(str_values.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x)));
else if (values is byte[] byte_values)
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(byte_values);
}
else if (values is Array array)
{
// array
var len = dtype.get_datatype_size() * (int)shape.size;
byte[] bytes = new byte[len];
System.Buffer.BlockCopy(array, 0, bytes, 0, len);
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
}
else
{ {
switch (values) switch (values)
{ {
@@ -166,32 +183,10 @@ namespace Tensorflow
case double val: case double val:
tensor_proto.DoubleVal.AddRange(new[] { val }); tensor_proto.DoubleVal.AddRange(new[] { val });
break; break;
case string val:
tensor_proto.StringVal.AddRange(val.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x.ToString())));
break;
default: default:
throw new Exception("make_tensor_proto Not Implemented"); throw new Exception("make_tensor_proto Not Implemented");
} }
} }
else if (dtype == TF_DataType.TF_STRING && !(values is NDArray))
{
if (values is string str)
{
tensor_proto.StringVal.Add(Google.Protobuf.ByteString.CopyFromUtf8(str));
}
else if (values is string[] str_values)
tensor_proto.StringVal.AddRange(str_values.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x)));
else if (values is byte[] byte_values)
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(byte_values);
}
else if (values is Array array)
{
// array
var len = dtype.get_datatype_size() * (int)shape.size;
byte[] bytes = new byte[len];
System.Buffer.BlockCopy(array, 0, bytes, 0, len);
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
}


return tensor_proto; return tensor_proto;
} }


+ 2
- 2
src/TensorFlowNET.Core/Training/Saving/Saver.cs View File

@@ -193,7 +193,7 @@ namespace Tensorflow


if (write_state) if (write_state)
{ {
var path = NDArray.AsStringArray(model_checkpoint_path[0])[0];
var path = model_checkpoint_path[0].StringData()[0];
_RecordLastCheckpoint(path); _RecordLastCheckpoint(path);
checkpoint_management.update_checkpoint_state_internal( checkpoint_management.update_checkpoint_state_internal(
save_dir: save_path_parent, save_dir: save_path_parent,
@@ -211,7 +211,7 @@ namespace Tensorflow
export_meta_graph(meta_graph_filename, strip_default_attrs: strip_default_attrs, save_debug_info: save_debug_info); export_meta_graph(meta_graph_filename, strip_default_attrs: strip_default_attrs, save_debug_info: save_debug_info);
} }


return _is_empty ? string.Empty : NDArray.AsStringArray(model_checkpoint_path[0])[0];
return _is_empty ? string.Empty : model_checkpoint_path[0].StringData()[0];
} }


public (Saver, object) import_meta_graph(string meta_graph_or_file, public (Saver, object) import_meta_graph(string meta_graph_or_file,


+ 2
- 2
src/TensorFlowNET.Core/ops.cs View File

@@ -165,8 +165,8 @@ namespace Tensorflow
if (dtype == TF_DataType.TF_STRING) if (dtype == TF_DataType.TF_STRING)
return ret; return ret;


if (dtype != ret.dtype)
ret = gen_math_ops.cast(ret, dtype.as_base_dtype(), name: name);
if (dtype.as_base_dtype() != ret.dtype.as_base_dtype())
ret = gen_math_ops.cast(ret, dtype, name: name);


return ret; return ret;
} }


+ 3
- 2
src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.index_directory.cs View File

@@ -3,6 +3,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using static Tensorflow.Binding;


namespace Tensorflow.Keras.Preprocessings namespace Tensorflow.Keras.Preprocessings
{ {
@@ -50,8 +51,8 @@ namespace Tensorflow.Keras.Preprocessings
if (!seed.HasValue) if (!seed.HasValue)
seed = np.random.randint((long)1e6); seed = np.random.randint((long)1e6);
var random_index = np.arange(label_list.Count); var random_index = np.arange(label_list.Count);
var rng = np.random.RandomState(seed.Value);
rng.shuffle(random_index);
tf.set_random_seed(seed.Value);
np.random.shuffle(random_index);
var index = random_index.ToArray<int>(); var index = random_index.ToArray<int>();


for (int i = 0; i < label_list.Count; i++) for (int i = 0; i < label_list.Count; i++)


+ 2
- 2
src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs View File

@@ -115,8 +115,8 @@ namespace Tensorflow.Keras
var start_positions = np.arange(0, num_seqs, sequence_stride); var start_positions = np.arange(0, num_seqs, sequence_stride);
if (shuffle) if (shuffle)
{ {
var rng = np.random.RandomState(seed);
rng.shuffle(start_positions);
tf.set_random_seed(seed);
np.random.shuffle(start_positions);
} }


var sequence_length_tensor = constant_op.constant(sequence_length, dtype: index_dtype); var sequence_length_tensor = constant_op.constant(sequence_length, dtype: index_dtype);


+ 27
- 0
test/TensorFlowNET.UnitTest/NumPy/Randomize.Test.cs View File

@@ -0,0 +1,27 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Tensorflow;
using Tensorflow.NumPy;

namespace TensorFlowNET.UnitTest.NumPy
{
/// <summary>
/// https://numpy.org/doc/1.20/reference/random/index.html
/// </summary>
[TestClass]
public class RandomizeTest : EagerModeTestBase
{
[TestMethod]
public void permutation()
{
var x = np.random.permutation(10);
Assert.AreEqual(x.shape, 10);
var y = np.random.permutation(x);
Assert.AreEqual(x.shape, 10);
Assert.AreNotEqual(x.ToArray<int>(), y.ToArray<int>());
}
}
}

Loading…
Cancel
Save