Added TokenToString conversion on model handle

LLama/Native/NativeApi.cs

@@ -312,5 +312,8 @@ namespace LLama.Native
 
         [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
         public static extern int llama_n_embd_from_model(SafeLlamaModelHandle model);
+
+        [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
+        public static extern byte* llama_token_to_str_with_model(SafeLlamaModelHandle safeLlamaModelHandle, int llamaToken);
     }
 }

LLama/Native/SafeLlamaModelHandle.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Drawing;
+using System.Text;
 using LLama.Exceptions;
 
 namespace LLama.Native
@@ -70,5 +72,42 @@ namespace LLama.Native
             if (err != 0)
                 throw new RuntimeError("Failed to apply lora adapter.");
         }
+
+        /// <summary>
+        /// Convert a single llama token into string bytes
+        /// </summary>
+        /// <param name="llama_token"></param>
+        /// <returns></returns>
+        public ReadOnlySpan<byte> TokenToSpan(int llama_token)
+        {
+            unsafe
+            {
+                var bytes = new ReadOnlySpan<byte>(NativeApi.llama_token_to_str_with_model(this, llama_token), int.MaxValue);
+                var terminator = bytes.IndexOf((byte)0);
+                return bytes.Slice(0, terminator);
+            }
+        }
+
+        /// <summary>
+        /// Convert a single llama token into a string
+        /// </summary>
+        /// <param name="llama_token"></param>
+        /// <param name="encoding">Encoding to use to decode the bytes into a string</param>
+        /// <returns></returns>
+        public string TokenToString(int llama_token, Encoding encoding)
+        {
+            var span = TokenToSpan(llama_token);
+
+            if (span.Length == 0)
+                return "";
+
+            unsafe
+            {
+                fixed (byte* ptr = &span[0])
+                {
+                    return encoding.GetString(ptr, span.Length);
+                }
+            }
+        }
     }
 }