Browse Source

np.permutation

tags/v0.60-tf.numpy
Oceania2018 4 years ago
parent
commit
a4a4da9d30
19 changed files with 113 additions and 65 deletions
  1. +4
    -4
      src/TensorFlowNET.Core/APIs/tf.math.cs
  2. +2
    -0
      src/TensorFlowNET.Core/Binding.Util.cs
  3. +3
    -0
      src/TensorFlowNET.Core/Contexts/ExecuteOpArgs.cs
  4. +1
    -1
      src/TensorFlowNET.Core/NumPy/Axis.cs
  5. +29
    -0
      src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs
  6. +2
    -0
      src/TensorFlowNET.Core/NumPy/NDArray.Index.cs
  7. +8
    -2
      src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs
  8. +1
    -1
      src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs
  9. +3
    -0
      src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs
  10. +1
    -3
      src/TensorFlowNET.Core/Numpy/NDArray.cs
  11. +1
    -22
      src/TensorFlowNET.Core/Numpy/Numpy.cs
  12. +3
    -0
      src/TensorFlowNET.Core/Sessions/BaseSession.cs
  13. +1
    -1
      src/TensorFlowNET.Core/Tensors/constant_op.cs
  14. +18
    -23
      src/TensorFlowNET.Core/Tensors/tensor_util.cs
  15. +2
    -2
      src/TensorFlowNET.Core/Training/Saving/Saver.cs
  16. +2
    -2
      src/TensorFlowNET.Core/ops.cs
  17. +3
    -2
      src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.index_directory.cs
  18. +2
    -2
      src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs
  19. +27
    -0
      test/TensorFlowNET.UnitTest/NumPy/Randomize.Test.cs

+ 4
- 4
src/TensorFlowNET.Core/APIs/tf.math.cs View File

@@ -21,6 +21,9 @@ namespace Tensorflow
public MathApi math { get; } = new MathApi();
public class MathApi
{
public Tensor argmax(Tensor input, Axis axis = null, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
=> gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);

public Tensor log(Tensor x, string name = null)
=> gen_math_ops.log(x, name);

@@ -539,15 +542,12 @@ namespace Tensorflow
public Tensor round(Tensor x, string name = null)
=> gen_math_ops.round(x, name: name);

public Tensor cast(Tensor x, TF_DataType dtype = TF_DataType.DtInvalid, string name = null)
public Tensor cast(Tensor x, TF_DataType dtype, string name = null)
=> math_ops.cast(x, dtype, name);

public Tensor cumsum(Tensor x, int axis = 0, bool exclusive = false, bool reverse = false, string name = null)
=> math_ops.cumsum(x, axis: axis, exclusive: exclusive, reverse: reverse, name: name);

public Tensor argmax(Tensor input, int axis = -1, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64)
=> gen_math_ops.arg_max(input, axis, name: name, output_type: output_type);

public Tensor square(Tensor x, string name = null)
=> gen_math_ops.square(x, name: name);
public Tensor squared_difference(Tensor x, Tensor y, string name = null)


+ 2
- 0
src/TensorFlowNET.Core/Binding.Util.cs View File

@@ -549,6 +549,8 @@ namespace Tensorflow
return tensors.dtype;
case IEnumerable<Tensor> tensors:
return tensors.First().dtype;
case RefVariable variable:
return variable.dtype;
case ResourceVariable variable:
return variable.dtype;
default:


+ 3
- 0
src/TensorFlowNET.Core/Contexts/ExecuteOpArgs.cs View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using static Tensorflow.Binding;

@@ -11,11 +12,13 @@ namespace Tensorflow
public object[] OpInputArgs { get; set; }
public Dictionary<string, object> OpAttrs { get; set; }
[DebuggerStepThrough]
public ExecuteOpArgs(params object[] inputArgs)
{
OpInputArgs = inputArgs;
}

[DebuggerStepThrough]
public ExecuteOpArgs SetAttributes(object attrs)
{
OpAttrs = ConvertToDict(attrs);


+ 1
- 1
src/TensorFlowNET.Core/NumPy/Axis.cs View File

@@ -56,7 +56,7 @@ namespace Tensorflow
=> constant_op.constant(axis);

public override string ToString()
=> $"({string.Join(", ", axis)})";
=> IsScalar ? $"{axis[0]}" : $"({string.Join(", ", axis)})";
}
}



+ 29
- 0
src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Text;

namespace Tensorflow.NumPy
{
public class RandomizedImpl
{
[AutoNumPy]
public NDArray permutation(int x) => new NDArray(random_ops.random_shuffle(math_ops.range(0, x)));

[AutoNumPy]
public NDArray permutation(NDArray x) => new NDArray(random_ops.random_shuffle(x));

[AutoNumPy]
public void shuffle(NDArray x)
{
var y = random_ops.random_shuffle(x);
Marshal.Copy(y.BufferToArray(), 0, x.TensorDataPointer, (int)x.bytesize);
}

public NDArray rand(params int[] shape)
=> throw new NotImplementedException("");

public NDArray randint(long x)
=> throw new NotImplementedException("");
}
}

+ 2
- 0
src/TensorFlowNET.Core/NumPy/NDArray.Index.cs View File

@@ -45,6 +45,8 @@ namespace Tensorflow.NumPy
{
if(mask.dtype == TF_DataType.TF_INT32)
return GetData(mask.ToArray<int>());
else if (mask.dtype == TF_DataType.TF_INT64)
return GetData(mask.ToArray<long>().Select(x => Convert.ToInt32(x)).ToArray());

throw new NotImplementedException("");
}


+ 8
- 2
src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs View File

@@ -17,7 +17,13 @@ namespace Tensorflow.NumPy
=> new NDArray(math_ops.argmax(a, axis ?? -1));

[AutoNumPy]
public static NDArray unique(NDArray a)
=> throw new NotImplementedException("");
public static (NDArray, NDArray) unique(NDArray a)
{
var(u, indice) = array_ops.unique(a);
return (new NDArray(u), new NDArray(indice));
}

[AutoNumPy]
public static void shuffle(NDArray x) => np.random.shuffle(x);
}
}

+ 1
- 1
src/TensorFlowNET.Core/NumPy/NumPy.Statistics.cs View File

@@ -13,6 +13,6 @@ namespace Tensorflow.NumPy
public static NDArray amin(NDArray x, int axis = 0) => new NDArray(tf.arg_min(x, axis));

[AutoNumPy]
public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.arg_max(x, axis));
public static NDArray amax(NDArray x, int axis = 0) => new NDArray(tf.math.argmax(x, axis));
}
}

+ 3
- 0
src/TensorFlowNET.Core/Numpy/NDArray.Creation.cs View File

@@ -25,6 +25,9 @@ namespace Tensorflow.NumPy
public NDArray(byte[] bytes, Shape shape, TF_DataType dtype)
: base(bytes, shape, dtype) { NewEagerTensorHandle(); }

public NDArray(long[] value, Shape? shape = null)
: base(value, shape) { NewEagerTensorHandle(); }

public NDArray(IntPtr address, Shape shape, TF_DataType dtype)
: base(address, shape, dtype) { NewEagerTensorHandle(); }



+ 1
- 3
src/TensorFlowNET.Core/Numpy/NDArray.cs View File

@@ -42,11 +42,9 @@ namespace Tensorflow.NumPy
public NDArray reshape(Shape newshape) => new NDArray(tf.reshape(this, newshape));
public NDArray astype(TF_DataType dtype) => new NDArray(math_ops.cast(this, dtype));
public NDArray ravel() => throw new NotImplementedException("");
public void shuffle(NDArray nd) => throw new NotImplementedException("");
public void shuffle(NDArray nd) => np.random.shuffle(nd);
public Array ToMuliDimArray<T>() => throw new NotImplementedException("");
public byte[] ToByteArray() => BufferToArray();
public static string[] AsStringArray(NDArray arr) => throw new NotImplementedException("");

public override string ToString() => NDArrayRender.ToString(this);
}
}

+ 1
- 22
src/TensorFlowNET.Core/Numpy/Numpy.cs View File

@@ -75,28 +75,7 @@ namespace Tensorflow.NumPy
public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8,
bool equal_nan = false) => throw new NotImplementedException("");

public static class random
{
public static NDArray permutation(int x)
{
throw new NotImplementedException("");
}

public static void shuffle(NDArray nd)
{

}

public static NDArray rand(params int[] shape)
=> throw new NotImplementedException("");

public static NDArray randint(long x)
=> throw new NotImplementedException("");

public static NDArray RandomState(int x)
=> throw new NotImplementedException("");
}

public static RandomizedImpl random = new RandomizedImpl();
public static LinearAlgebraImpl linalg = new LinearAlgebraImpl();
}
}

+ 3
- 0
src/TensorFlowNET.Core/Sessions/BaseSession.cs View File

@@ -193,6 +193,9 @@ namespace Tensorflow
case double v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v));
break;
case string v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v));
break;
case Array v:
feeds[i++] = new KeyValuePair<TF_Output, Tensor>(key._as_tf_output(), new Tensor(v, v.GetShape()));
break;


+ 1
- 1
src/TensorFlowNET.Core/Tensors/constant_op.cs View File

@@ -117,7 +117,7 @@ namespace Tensorflow
case Shape val:
return new EagerTensor(val.dims, new Shape(val.ndim));
case Axis val:
return new EagerTensor(val.axis, new Shape(val.size));
return new EagerTensor(val.axis, val.IsScalar ? Shape.Scalar : new Shape(val.size));
case string val:
return new EagerTensor(new[] { val }, Shape.Scalar);
case string[] val:


+ 18
- 23
src/TensorFlowNET.Core/Tensors/tensor_util.cs View File

@@ -141,7 +141,24 @@ namespace Tensorflow
byte[] bytes = nd.ToByteArray();
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
}
else if (!values.GetType().IsArray)
else if (dtype == TF_DataType.TF_STRING && !(values is NDArray))
{
if (values is string str)
tensor_proto.StringVal.Add(Google.Protobuf.ByteString.CopyFromUtf8(str));
else if (values is string[] str_values)
tensor_proto.StringVal.AddRange(str_values.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x)));
else if (values is byte[] byte_values)
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(byte_values);
}
else if (values is Array array)
{
// array
var len = dtype.get_datatype_size() * (int)shape.size;
byte[] bytes = new byte[len];
System.Buffer.BlockCopy(array, 0, bytes, 0, len);
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
}
else
{
switch (values)
{
@@ -166,32 +183,10 @@ namespace Tensorflow
case double val:
tensor_proto.DoubleVal.AddRange(new[] { val });
break;
case string val:
tensor_proto.StringVal.AddRange(val.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x.ToString())));
break;
default:
throw new Exception("make_tensor_proto Not Implemented");
}
}
else if (dtype == TF_DataType.TF_STRING && !(values is NDArray))
{
if (values is string str)
{
tensor_proto.StringVal.Add(Google.Protobuf.ByteString.CopyFromUtf8(str));
}
else if (values is string[] str_values)
tensor_proto.StringVal.AddRange(str_values.Select(x => Google.Protobuf.ByteString.CopyFromUtf8(x)));
else if (values is byte[] byte_values)
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(byte_values);
}
else if (values is Array array)
{
// array
var len = dtype.get_datatype_size() * (int)shape.size;
byte[] bytes = new byte[len];
System.Buffer.BlockCopy(array, 0, bytes, 0, len);
tensor_proto.TensorContent = Google.Protobuf.ByteString.CopyFrom(bytes);
}

return tensor_proto;
}


+ 2
- 2
src/TensorFlowNET.Core/Training/Saving/Saver.cs View File

@@ -193,7 +193,7 @@ namespace Tensorflow

if (write_state)
{
var path = NDArray.AsStringArray(model_checkpoint_path[0])[0];
var path = model_checkpoint_path[0].StringData()[0];
_RecordLastCheckpoint(path);
checkpoint_management.update_checkpoint_state_internal(
save_dir: save_path_parent,
@@ -211,7 +211,7 @@ namespace Tensorflow
export_meta_graph(meta_graph_filename, strip_default_attrs: strip_default_attrs, save_debug_info: save_debug_info);
}

return _is_empty ? string.Empty : NDArray.AsStringArray(model_checkpoint_path[0])[0];
return _is_empty ? string.Empty : model_checkpoint_path[0].StringData()[0];
}

public (Saver, object) import_meta_graph(string meta_graph_or_file,


+ 2
- 2
src/TensorFlowNET.Core/ops.cs View File

@@ -165,8 +165,8 @@ namespace Tensorflow
if (dtype == TF_DataType.TF_STRING)
return ret;

if (dtype != ret.dtype)
ret = gen_math_ops.cast(ret, dtype.as_base_dtype(), name: name);
if (dtype.as_base_dtype() != ret.dtype.as_base_dtype())
ret = gen_math_ops.cast(ret, dtype, name: name);

return ret;
}


+ 3
- 2
src/TensorFlowNET.Keras/Preprocessings/DatasetUtils.index_directory.cs View File

@@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using static Tensorflow.Binding;

namespace Tensorflow.Keras.Preprocessings
{
@@ -50,8 +51,8 @@ namespace Tensorflow.Keras.Preprocessings
if (!seed.HasValue)
seed = np.random.randint((long)1e6);
var random_index = np.arange(label_list.Count);
var rng = np.random.RandomState(seed.Value);
rng.shuffle(random_index);
tf.set_random_seed(seed.Value);
np.random.shuffle(random_index);
var index = random_index.ToArray<int>();

for (int i = 0; i < label_list.Count; i++)


+ 2
- 2
src/TensorFlowNET.Keras/Preprocessings/Preprocessing.image_dataset_from_directory.cs View File

@@ -115,8 +115,8 @@ namespace Tensorflow.Keras
var start_positions = np.arange(0, num_seqs, sequence_stride);
if (shuffle)
{
var rng = np.random.RandomState(seed);
rng.shuffle(start_positions);
tf.set_random_seed(seed);
np.random.shuffle(start_positions);
}

var sequence_length_tensor = constant_op.constant(sequence_length, dtype: index_dtype);


+ 27
- 0
test/TensorFlowNET.UnitTest/NumPy/Randomize.Test.cs View File

@@ -0,0 +1,27 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Tensorflow;
using Tensorflow.NumPy;

namespace TensorFlowNET.UnitTest.NumPy
{
/// <summary>
/// https://numpy.org/doc/1.20/reference/random/index.html
/// </summary>
[TestClass]
public class RandomizeTest : EagerModeTestBase
{
[TestMethod]
public void permutation()
{
var x = np.random.permutation(10);
Assert.AreEqual(x.shape, 10);
var y = np.random.permutation(x);
Assert.AreEqual(x.shape, 10);
Assert.AreNotEqual(x.ToArray<int>(), y.ToArray<int>());
}
}
}

Loading…
Cancel
Save