diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs
index 5218d55c..ed6b0e5a 100644
--- a/LLama/Native/NativeApi.cs
+++ b/LLama/Native/NativeApi.cs
@@ -312,5 +312,8 @@ namespace LLama.Native
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
public static extern int llama_n_embd_from_model(SafeLlamaModelHandle model);
+
+ [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
+ public static extern byte* llama_token_to_str_with_model(SafeLlamaModelHandle safeLlamaModelHandle, int llamaToken);
}
}
diff --git a/LLama/Native/SafeLlamaModelHandle.cs b/LLama/Native/SafeLlamaModelHandle.cs
index e047c8fe..939fc57d 100644
--- a/LLama/Native/SafeLlamaModelHandle.cs
+++ b/LLama/Native/SafeLlamaModelHandle.cs
@@ -1,4 +1,6 @@
using System;
+using System.Drawing;
+using System.Text;
using LLama.Exceptions;
namespace LLama.Native
@@ -70,5 +72,42 @@ namespace LLama.Native
if (err != 0)
throw new RuntimeError("Failed to apply lora adapter.");
}
+
+ ///
+ /// Convert a single llama token into string bytes
+ ///
+ ///
+ ///
+ public ReadOnlySpan TokenToSpan(int llama_token)
+ {
+ unsafe
+ {
+ var bytes = new ReadOnlySpan(NativeApi.llama_token_to_str_with_model(this, llama_token), int.MaxValue);
+ var terminator = bytes.IndexOf((byte)0);
+ return bytes.Slice(0, terminator);
+ }
+ }
+
+ ///
+ /// Convert a single llama token into a string
+ ///
+ ///
+ /// Encoding to use to decode the bytes into a string
+ ///
+ public string TokenToString(int llama_token, Encoding encoding)
+ {
+ var span = TokenToSpan(llama_token);
+
+ if (span.Length == 0)
+ return "";
+
+ unsafe
+ {
+ fixed (byte* ptr = &span[0])
+ {
+ return encoding.GetString(ptr, span.Length);
+ }
+ }
+ }
}
}