From 3c5547b2b752c3349d3eecfc8bacb5b93793f323 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Wed, 25 Oct 2023 22:29:19 +0100 Subject: [PATCH] Reduced some uses of `NativeApi` in `BatchedDecoding` by adding some helper methods --- LLama.Examples/NewVersion/BatchedDecoding.cs | 64 ++++++++------------ LLama/LLamaWeights.cs | 15 +++++ LLama/Native/LLamaTokenDataArray.cs | 43 +++++++++++++ 3 files changed, 84 insertions(+), 38 deletions(-) diff --git a/LLama.Examples/NewVersion/BatchedDecoding.cs b/LLama.Examples/NewVersion/BatchedDecoding.cs index 7d140e81..66929310 100644 --- a/LLama.Examples/NewVersion/BatchedDecoding.cs +++ b/LLama.Examples/NewVersion/BatchedDecoding.cs @@ -11,6 +11,10 @@ public class BatchedDecoding private const int n_parallel = 8; private const int n_len = 32; + private const int top_k = 40; + private const float top_p = 0.9f; + private const float temp = 0.4f; + public static async Task Run() { Console.Write("Please input your model path: "); @@ -91,8 +95,8 @@ public class BatchedDecoding for (var i = 0; i < n_parallel; i++) streams[i] = new(); - var eos = NativeApi.llama_token_eos(model.NativeHandle); - var nl = NativeApi.llama_token_nl(model.NativeHandle); + var eos = model.EndOfSentenceToken; + var nl = model.NewlineToken; var timer = new Stopwatch(); timer.Start(); @@ -106,50 +110,34 @@ public class BatchedDecoding if (i_batch[i] < 0) continue; + var n_vocab = model.VocabCount; + LLamaTokenDataArray candidates; unsafe { - var n_vocab = model.VocabCount; - var logits = NativeApi.llama_get_logits_ith(context.NativeHandle, i_batch[i]); - - var candidates = new LLamaTokenData[n_vocab]; - for (var token_id = 0; token_id < n_vocab; token_id++) - { - candidates[token_id] = new LLamaTokenData - { - id = token_id, - logit = logits[token_id] - }; - } - - var candidates_p = new LLamaTokenDataArray(candidates); - using var pin = LLamaTokenDataArrayNative.Create(candidates_p, out var candidates_native); - - const int top_k = 40; - const float top_p = 0.9f; - const float temp = 0.4f; - - NativeApi.llama_sample_top_k(context.NativeHandle, ref candidates_native, top_k, 1); - NativeApi.llama_sample_top_p(context.NativeHandle, ref candidates_native, top_p, 1); - NativeApi.llama_sample_temperature(context.NativeHandle, ref candidates_native, temp); + candidates = LLamaTokenDataArray.Create(new Span(NativeApi.llama_get_logits_ith(context.NativeHandle, i_batch[i]), n_vocab)); + } + using var pin = LLamaTokenDataArrayNative.Create(candidates, out var candidates_native); - var new_token_id = NativeApi.llama_sample_token(context.NativeHandle, ref candidates_native); + candidates_native.TopK(context.NativeHandle, top_k); + candidates_native.TopP(context.NativeHandle, top_p); + candidates_native.Temperature(context.NativeHandle, temp); + var new_token_id = candidates_native.SampleToken(context.NativeHandle); - if (new_token_id == eos || new_token_id == nl) - { - i_batch[i] = -1; - Console.WriteLine($"Completed Stream {i} early"); - continue; - } + if (new_token_id == eos || new_token_id == nl) + { + i_batch[i] = -1; + Console.WriteLine($"Completed Stream {i} early"); + continue; + } - streams[i].Add(new_token_id); + streams[i].Add(new_token_id); - i_batch[i] = batch.NativeBatch.n_tokens; + i_batch[i] = batch.NativeBatch.n_tokens; - // push this new token for next evaluation - llama_batch_add(batch, new_token_id, n_cur, new() { (LLamaSeqId)i }, true); + // push this new token for next evaluation + llama_batch_add(batch, new_token_id, n_cur, new() { (LLamaSeqId)i }, true); - n_decode++; - } + n_decode++; } // all streams are finished diff --git a/LLama/LLamaWeights.cs b/LLama/LLamaWeights.cs index 64878e2a..7ae104a5 100644 --- a/LLama/LLamaWeights.cs +++ b/LLama/LLamaWeights.cs @@ -38,6 +38,21 @@ namespace LLama /// public ulong ParameterCount => NativeHandle.ParameterCount; + /// + /// Get the newline token for this model + /// + public int NewlineToken => NativeApi.llama_token_nl(NativeHandle); + + /// + /// Get the "end of sentence" token for this model + /// + public int EndOfSentenceToken => NativeApi.llama_token_eos(NativeHandle); + + /// + /// Get the "beginning of sentence" token for this model + /// + public int BeginningOfSentenceToken => NativeApi.llama_token_bos(NativeHandle); + /// /// Dimension of embedding vectors /// diff --git a/LLama/Native/LLamaTokenDataArray.cs b/LLama/Native/LLamaTokenDataArray.cs index 7a2965ed..22235a79 100644 --- a/LLama/Native/LLamaTokenDataArray.cs +++ b/LLama/Native/LLamaTokenDataArray.cs @@ -96,5 +96,48 @@ namespace LLama.Native return handle; } + + /// + /// Perform TopK sampling, sorting the data and reducing the size to k + /// + /// + /// Number of tokens to keep + /// Minimum number to keep + public void TopK(SafeLLamaContextHandle context, int k, ulong minKeep = 1) + { + NativeApi.llama_sample_top_k(context, ref this, k, minKeep); + } + + /// + /// Perform top p sampling, sorting the data and keeping only logits more likely than p + /// + /// + /// + /// + public void TopP(SafeLLamaContextHandle context, float p, ulong minKeep = 1) + { + NativeApi.llama_sample_top_p(context, ref this, p, minKeep); + } + + /// + /// Apply temperature to logits + /// + /// + /// + public void Temperature(SafeLLamaContextHandle context, float temp) + { + NativeApi.llama_sample_temperature(context, ref this, temp); + } + + /// + /// Sample a token from the set of possible tokens + /// + /// + /// + /// + public int SampleToken(SafeLLamaContextHandle context) + { + return NativeApi.llama_sample_token(context, ref this); + } } }