From 31287b5e6e58488f347faef8b2a7ccb8557144d7 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Fri, 25 Aug 2023 16:58:19 +0100 Subject: [PATCH] Rewritten TokenToSpan/TokenToString to better fit the new way it's done in llama.cpp with a few different options: - Just convert it to a `string`, nice and simple - Write the bytes to a `Span` no allocations - Write the chars to a `StringBuilder` potentially no allocations --- LLama/LLamaContext.cs | 7 ++-- LLama/LLamaInstructExecutor.cs | 8 ++-- LLama/LLamaInteractExecutor.cs | 10 ++--- LLama/Native/SafeLLamaContextHandle.cs | 24 +++++++++--- LLama/Native/SafeLlamaModelHandle.cs | 54 ++++++++++++++++++++------ 5 files changed, 74 insertions(+), 29 deletions(-) diff --git a/LLama/LLamaContext.cs b/LLama/LLamaContext.cs index 3f27d1ba..0cb77f60 100644 --- a/LLama/LLamaContext.cs +++ b/LLama/LLamaContext.cs @@ -132,9 +132,10 @@ namespace LLama /// public string DeTokenize(IEnumerable tokens) { - StringBuilder sb = new(); - foreach(var token in tokens) - sb.Append(_ctx.TokenToString(token, _encoding)); + var sb = new StringBuilder(); + foreach (var token in tokens) + _ctx.TokenToString(token, _encoding, sb); + return sb.ToString(); } diff --git a/LLama/LLamaInstructExecutor.cs b/LLama/LLamaInstructExecutor.cs index bcbc2998..a6c8603c 100644 --- a/LLama/LLamaInstructExecutor.cs +++ b/LLama/LLamaInstructExecutor.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -141,9 +142,10 @@ namespace LLama { if (args.Antiprompts is not null && args.Antiprompts.Count > 0) { - string last_output = ""; - foreach (var id in _last_n_tokens) - last_output += Context.NativeHandle.TokenToString(id, Context.Encoding); + var last_output_builder = new StringBuilder(); + foreach (var token in _last_n_tokens) + Context.NativeHandle.TokenToString(token, Context.Encoding, last_output_builder); + var last_output = last_output_builder.ToString(); foreach (var antiprompt in args.Antiprompts) { diff --git a/LLama/LLamaInteractExecutor.cs b/LLama/LLamaInteractExecutor.cs index aa184ca3..9b57e74f 100644 --- a/LLama/LLamaInteractExecutor.cs +++ b/LLama/LLamaInteractExecutor.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text; namespace LLama { @@ -132,11 +133,10 @@ namespace LLama { if (args.Antiprompts is not null && args.Antiprompts.Count > 0) { - string last_output = ""; - foreach (var id in _last_n_tokens) - { - last_output += Context.NativeHandle.TokenToString(id, Context.Encoding); - } + var last_output_builder = new StringBuilder(); + foreach (var token in _last_n_tokens) + Context.NativeHandle.TokenToString(token, Context.Encoding, last_output_builder); + var last_output = last_output_builder.ToString(); foreach (var antiprompt in args.Antiprompts) { diff --git a/LLama/Native/SafeLLamaContextHandle.cs b/LLama/Native/SafeLLamaContextHandle.cs index 86c1c71c..aa9c9439 100644 --- a/LLama/Native/SafeLLamaContextHandle.cs +++ b/LLama/Native/SafeLLamaContextHandle.cs @@ -183,7 +183,7 @@ namespace LLama.Native /// /// Convert a token into a string /// - /// + /// Token to decode into a string /// /// public string TokenToString(int token, Encoding encoding) @@ -192,13 +192,25 @@ namespace LLama.Native } /// - /// Convert a token into a span of bytes that could be decoded into a string + /// Append a single llama token to a string builder /// - /// - /// - public ReadOnlySpan TokenToSpan(int token) + /// Token to decode + /// + /// string builder to append the result to + public void TokenToString(int token, Encoding encoding, StringBuilder dest) + { + ThrowIfDisposed().TokenToString(token, encoding, dest); + } + + /// + /// Convert a single llama token into bytes + /// + /// Token to decode + /// A span to attempt to write into. If this is too small nothing will be written + /// The size of this token. **nothing will be written** if this is larger than `dest` + public int TokenToSpan(int token, Span dest) { - return ThrowIfDisposed().TokenToSpan(token); + return ThrowIfDisposed().TokenToSpan(token, dest); } /// diff --git a/LLama/Native/SafeLlamaModelHandle.cs b/LLama/Native/SafeLlamaModelHandle.cs index 059bb070..976d32e8 100644 --- a/LLama/Native/SafeLlamaModelHandle.cs +++ b/LLama/Native/SafeLlamaModelHandle.cs @@ -83,24 +83,20 @@ namespace LLama.Native #region tokenize /// - /// Convert a single llama token into string bytes + /// Convert a single llama token into bytes /// - /// - /// - public ReadOnlySpan TokenToSpan(int llama_token) + /// Token to decode + /// A span to attempt to write into. If this is too small nothing will be written + /// The size of this token. **nothing will be written** if this is larger than `dest` + public int TokenToSpan(int llama_token, Span dest) { unsafe { - var length = NativeApi.llama_token_to_str_with_model(this, llama_token, null, 0); - var bytes = new byte[-length]; - - fixed (byte* bytePtr = bytes) + fixed (byte* destPtr = dest) { - var written = NativeApi.llama_token_to_str_with_model(this, llama_token, bytePtr, bytes.Length); - Debug.Assert(written == bytes.Length); + var length = NativeApi.llama_token_to_str_with_model(this, llama_token, destPtr, dest.Length); + return Math.Abs(length); } - - return new ReadOnlySpan(bytes); } } @@ -130,6 +126,40 @@ namespace LLama.Native } } + /// + /// Append a single llama token to a string builder + /// + /// Token to decode + /// + /// string builder to append the result to + public void TokenToString(int llama_token, Encoding encoding, StringBuilder dest) + { + unsafe + { + var length = NativeApi.llama_token_to_str_with_model(this, llama_token, null, 0); + if (length == 0) + return; + + Span bytes = stackalloc byte[-length]; + fixed (byte* bytePtr = bytes) + { + // Decode into bytes + var written = NativeApi.llama_token_to_str_with_model(this, llama_token, bytePtr, bytes.Length); + Debug.Assert(written == bytes.Length); + + // Decode into chars + var charCount = encoding.GetCharCount(bytePtr, bytes.Length); + Span chars = stackalloc char[charCount]; + fixed (char* charPtr = chars) + encoding.GetChars(bytePtr, bytes.Length, charPtr, chars.Length); + + // Write it to the output + for (var i = 0; i < chars.Length; i++) + dest.Append(chars[i]); + } + } + } + /// /// Convert a string of text into tokens ///