Add Tensorflow.Text project.

TensorFlow.NET.sln

@@ -9,10 +9,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Benchmark", "src
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.UnitTest", "test\TensorFlowNET.UnitTest\Tensorflow.UnitTest.csproj", "{23C28035-2FCE-41F3-9A12-E73CE8A5AE32}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Console", "src\TensorFlowNET.Console\TensorFlowNET.Console.csproj", "{03F06299-3F4B-4449-A709-3A647657BC0C}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Console", "src\TensorFlowNET.Console\Tensorflow.Console.csproj", "{03F06299-3F4B-4449-A709-3A647657BC0C}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tensorflow.Keras", "src\TensorFlowNET.Keras\Tensorflow.Keras.csproj", "{49D71826-C03D-4FA7-9BAC-22C1327E65CF}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tensorflow.Text", "src\TensorFlowNET.Text\Tensorflow.Text.csproj", "{1AB8108D-4FFE-4A16-88E7-328EAF686370}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -149,6 +151,30 @@ Global
 		{49D71826-C03D-4FA7-9BAC-22C1327E65CF}.Release|x64.Build.0 = Release|x64
 		{49D71826-C03D-4FA7-9BAC-22C1327E65CF}.Release|x86.ActiveCfg = Release|Any CPU
 		{49D71826-C03D-4FA7-9BAC-22C1327E65CF}.Release|x86.Build.0 = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|x64.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug|x86.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|Any CPU.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|Any CPU.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|x64.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|x64.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|x86.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Debug-Minimal|x86.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|x64.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|x64.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|x86.ActiveCfg = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Publish|x86.Build.0 = Debug|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|x64.ActiveCfg = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|x64.Build.0 = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|x86.ActiveCfg = Release|Any CPU
+		{1AB8108D-4FFE-4A16-88E7-328EAF686370}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/TensorFlowNET.Console/Exploring.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+using static Tensorflow.TextApi;
+
+namespace Tensorflow
+{
+    public class Exploring
+    {
+        public void Run()
+        {
+            var docs = tf.constant(new[] { "Everything not saved will be lost." });
+            var tokenizer = text.WhitespaceTokenizer();
+            text.wordshape(docs, Text.WordShape.HAS_TITLE_CASE);
+
+            throw new NotImplementedException("");
+        }
+    }
+}

src/TensorFlowNET.Console/Program.cs

@@ -6,6 +6,10 @@ namespace Tensorflow
     {
         static void Main(string[] args)
         {
+            // this class is used explor new features.
+            var exploring = new Exploring();
+            // exploring.Run();
+
             // boot .net core 10.5M.
             var mm = new MemoryMonitor();
             // warm up tensorflow.net 28.5M.

src/TensorFlowNET.Console/TensorFlowNET.Console.csproj → src/TensorFlowNET.Console/Tensorflow.Console.csproj

@@ -8,11 +8,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.3.0" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.3.1" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\TensorFlowNET.Keras\Tensorflow.Keras.csproj" />
+    <ProjectReference Include="..\TensorFlowNET.Text\Tensorflow.Text.csproj" />
   </ItemGroup>
 
 </Project>

src/TensorFlowNET.Core/Tensorflow.Binding.csproj

@@ -82,7 +82,6 @@ TensorFlow .NET v0.30 is focused on making more Keras API work including:
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.11.4" />
     <PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.138" />
     <PackageReference Include="NumSharp.Lite" Version="0.1.9" />
     <PackageReference Include="Protobuf.Text" Version="0.4.0" />

src/TensorFlowNET.Text/Enums/Reduction.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text
+{
+    public enum Reduction
+    {
+        None,
+        STRING_JOIN
+    }
+}

src/TensorFlowNET.Text/Enums/WordShape.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text
+{
+    public enum WordShape
+    {
+        HAS_TITLE_CASE,
+        IS_UPPERCASE,
+        HAS_SOME_PUNCT_OR_SYMBOL,
+        IS_NUMERIC_VALUE
+    }
+}

src/TensorFlowNET.Text/Operations/TextOps.ngrams.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text
+{
+    public partial class TextOps
+    {
+        public static Tensor ngrams(Tensor input, int width, 
+            int axis = -1, 
+            Reduction reduction_type = Reduction.None,
+            string string_separator = " ",
+            string name = null)
+            => throw new NotImplementedException("");
+    }
+}

src/TensorFlowNET.Text/Operations/TextOps.wordshape.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text
+{
+    public partial class TextOps
+    {
+        public static Tensor wordshape(Tensor input, WordShape pattern, string name = null)
+            => throw new NotImplementedException("");
+    }
+}

src/TensorFlowNET.Text/Tensorflow.Text.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <RootNamespace>Tensorflow.Text</RootNamespace>
+    <AssemblyName>Tensorflow.Text</AssemblyName>
+    <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+    <Version>0.0.1</Version>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
+  </ItemGroup>
+
+</Project>

src/TensorFlowNET.Text/TextApi.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Text;
+
+namespace Tensorflow
+{
+    public class TextApi
+    {
+        public static TextInterface text { get; } = new TextInterface();
+    }
+}

src/TensorFlowNET.Text/TextInterface.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Text.Tokenizers;
+
+namespace Tensorflow.Text
+{
+    public class TextInterface
+    {
+        public ITokenizer WhitespaceTokenizer()
+            => new WhitespaceTokenizer();
+
+        public Tensor wordshape(Tensor input, WordShape pattern, string name = null)
+            => TextOps.wordshape(input, pattern, name: name);
+
+        /// <summary>
+        /// Create a tensor of n-grams based on the input data `data`.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <param name="width"></param>
+        /// <param name="axis"></param>
+        /// <param name="reduction_type"></param>
+        /// <param name="string_separator"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public static Tensor ngrams(Tensor input, int width,
+            int axis = -1,
+            Reduction reduction_type = Reduction.None,
+            string string_separator = " ",
+            string name = null)
+            => TextOps.ngrams(input, width,
+                axis: axis,
+                reduction_type: reduction_type,
+                string_separator: string_separator,
+                name: name);
+    }
+}

src/TensorFlowNET.Text/Tokenizers/ITokenizer.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text.Tokenizers
+{
+    public interface ITokenizer
+    {
+        Tensor tokenize(Tensor input);
+    }
+}

src/TensorFlowNET.Text/Tokenizers/UnicodeScriptTokenizer.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text.Tokenizers
+{
+    public class UnicodeScriptTokenizer : ITokenizer
+    {
+        public Tensor tokenize(Tensor input)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/TensorFlowNET.Text/Tokenizers/WhitespaceTokenizer.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Text.Tokenizers
+{
+    public class WhitespaceTokenizer : ITokenizer
+    {
+        /// <summary>
+        /// Tokenizes a tensor of UTF-8 strings on whitespaces.
+        /// </summary>
+        /// <param name="input"></param>
+        /// <returns></returns>
+        public Tensor tokenize(Tensor input)
+        {
+            throw new NotImplementedException("");
+        }
+    }
+}

test/TensorFlowNET.UnitTest/Tensorflow.UnitTest.csproj

@@ -48,14 +48,15 @@
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="5.10.3" />
     <PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.138" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.1.2" />
     <PackageReference Include="MSTest.TestFramework" Version="2.1.2" />
-    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.3.0" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.3.1" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\TensorFlowNET.Keras\Tensorflow.Keras.csproj" />
+    <ProjectReference Include="..\..\src\TensorFlowNET.Text\Tensorflow.Text.csproj" />
   </ItemGroup>
 
   <ItemGroup>

test/TensorFlowNET.UnitTest/Text/TokenizerTest.cs

@@ -0,0 +1,19 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+using static Tensorflow.TextApi;
+
+namespace TensorFlowNET.UnitTest.Text
+{
+    [TestClass]
+    public class TokenizerTest
+    {
+        [TestMethod]
+        public void Tokenize()
+        {
+            var docs = tf.constant(new[] { "Everything not saved will be lost." });
+        }
+    }
+}