Rewritten TokenToSpan/TokenToString to better fit the new way it's done in llama.cpp with a few different options:

- Just convert it to a `string`, nice and simple
 - Write the bytes to a `Span<byte>` no allocations
 - Write the chars to a `StringBuilder` potentially no allocations

LLama/LLamaContext.cs

@@ -132,9 +132,10 @@ namespace LLama
         /// <returns></returns>
         public string DeTokenize(IEnumerable<llama_token> tokens)
         {
-            StringBuilder sb = new();
-            foreach(var token in tokens)
-                sb.Append(_ctx.TokenToString(token, _encoding));
+            var sb = new StringBuilder();
+            foreach (var token in tokens)
+                _ctx.TokenToString(token, _encoding, sb);
+
             return sb.ToString();
         }
 

LLama/LLamaInstructExecutor.cs

@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
@@ -141,9 +142,10 @@ namespace LLama
             {
                 if (args.Antiprompts is not null && args.Antiprompts.Count > 0)
                 {
-                    string last_output = "";
-                    foreach (var id in _last_n_tokens)
-                        last_output += Context.NativeHandle.TokenToString(id, Context.Encoding);
+                    var last_output_builder = new StringBuilder();
+                    foreach (var token in _last_n_tokens)
+                        Context.NativeHandle.TokenToString(token, Context.Encoding, last_output_builder);
+                    var last_output = last_output_builder.ToString();
 
                     foreach (var antiprompt in args.Antiprompts)
                     {

LLama/LLamaInteractExecutor.cs

@@ -7,6 +7,7 @@ using System.IO;
 using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text;
 
 namespace LLama
 {
@@ -132,11 +133,10 @@ namespace LLama
             {
                 if (args.Antiprompts is not null && args.Antiprompts.Count > 0)
                 {
-                    string last_output = "";
-                    foreach (var id in _last_n_tokens)
-                    {
-                        last_output += Context.NativeHandle.TokenToString(id, Context.Encoding);
-                    }
+                    var last_output_builder = new StringBuilder();
+                    foreach (var token in _last_n_tokens)
+                        Context.NativeHandle.TokenToString(token, Context.Encoding, last_output_builder);
+                    var last_output = last_output_builder.ToString();
 
                     foreach (var antiprompt in args.Antiprompts)
                     {

LLama/Native/SafeLLamaContextHandle.cs

@@ -183,7 +183,7 @@ namespace LLama.Native
         /// <summary>
         /// Convert a token into a string
         /// </summary>
-        /// <param name="token"></param>
+        /// <param name="token">Token to decode into a string</param>
         /// <param name="encoding"></param>
         /// <returns></returns>
         public string TokenToString(int token, Encoding encoding)
@@ -192,13 +192,25 @@ namespace LLama.Native
         }
 
         /// <summary>
-        /// Convert a token into a span of bytes that could be decoded into a string
+        /// Append a single llama token to a string builder
         /// </summary>
-        /// <param name="token"></param>
-        /// <returns></returns>
-        public ReadOnlySpan<byte> TokenToSpan(int token)
+        /// <param name="token">Token to decode</param>
+        /// <param name="encoding"></param>
+        /// <param name="dest">string builder to append the result to</param>
+        public void TokenToString(int token, Encoding encoding, StringBuilder dest)
+        {
+            ThrowIfDisposed().TokenToString(token, encoding, dest);
+        }
+
+        /// <summary>
+        /// Convert a single llama token into bytes
+        /// </summary>
+        /// <param name="token">Token to decode</param>
+        /// <param name="dest">A span to attempt to write into. If this is too small nothing will be written</param>
+        /// <returns>The size of this token. **nothing will be written** if this is larger than `dest`</returns>
+        public int TokenToSpan(int token, Span<byte> dest)
         {
-            return ThrowIfDisposed().TokenToSpan(token);
+            return ThrowIfDisposed().TokenToSpan(token, dest);
         }
 
         /// <summary>

LLama/Native/SafeLlamaModelHandle.cs

@@ -83,24 +83,20 @@ namespace LLama.Native
 
         #region tokenize
         /// <summary>
-        /// Convert a single llama token into string bytes
+        /// Convert a single llama token into bytes
         /// </summary>
-        /// <param name="llama_token"></param>
-        /// <returns></returns>
-        public ReadOnlySpan<byte> TokenToSpan(int llama_token)
+        /// <param name="llama_token">Token to decode</param>
+        /// <param name="dest">A span to attempt to write into. If this is too small nothing will be written</param>
+        /// <returns>The size of this token. **nothing will be written** if this is larger than `dest`</returns>
+        public int TokenToSpan(int llama_token, Span<byte> dest)
         {
             unsafe
             {
-                var length = NativeApi.llama_token_to_str_with_model(this, llama_token, null, 0);
-                var bytes = new byte[-length];
-
-                fixed (byte* bytePtr = bytes)
+                fixed (byte* destPtr = dest)
                 {
-                    var written = NativeApi.llama_token_to_str_with_model(this, llama_token, bytePtr, bytes.Length);
-                    Debug.Assert(written == bytes.Length);
+                    var length = NativeApi.llama_token_to_str_with_model(this, llama_token, destPtr, dest.Length);
+                    return Math.Abs(length);
                 }
-
-                return new ReadOnlySpan<byte>(bytes);
             }
         }
 
@@ -130,6 +126,40 @@ namespace LLama.Native
             }
         }
 
+        /// <summary>
+        /// Append a single llama token to a string builder
+        /// </summary>
+        /// <param name="llama_token">Token to decode</param>
+        /// <param name="encoding"></param>
+        /// <param name="dest">string builder to append the result to</param>
+        public void TokenToString(int llama_token, Encoding encoding, StringBuilder dest)
+        {
+            unsafe
+            {
+                var length = NativeApi.llama_token_to_str_with_model(this, llama_token, null, 0);
+                if (length == 0)
+                    return;
+
+                Span<byte> bytes = stackalloc byte[-length];
+                fixed (byte* bytePtr = bytes)
+                {
+                    // Decode into bytes
+                    var written = NativeApi.llama_token_to_str_with_model(this, llama_token, bytePtr, bytes.Length);
+                    Debug.Assert(written == bytes.Length);
+
+                    // Decode into chars
+                    var charCount = encoding.GetCharCount(bytePtr, bytes.Length);
+                    Span<char> chars = stackalloc char[charCount];
+                    fixed (char* charPtr = chars)
+                        encoding.GetChars(bytePtr, bytes.Length, charPtr, chars.Length);
+
+                    // Write it to the output
+                    for (var i = 0; i < chars.Length; i++)
+                        dest.Append(chars[i]);
+                }
+            }
+        }
+
         /// <summary>
         /// Convert a string of text into tokens
         /// </summary>