From 08f1615e602797e1e6b681b5857ad65edb9eca3f Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Sat, 23 Sep 2023 15:22:57 +0100 Subject: [PATCH 1/3] - Converted LLamaStatelessExecutor to run `Exec` calls inside an awaited task. This unblocks async callers while the model is being evaluated. - Added a "spinner" to the `StatelessModeExecute` demo, which spins while waiting for the next token (demonstrating that it's not blocked). --- .../NewVersion/StatelessModeExecute.cs | 40 ++++++++++++++++++- LLama/Extensions/IReadOnlyListExtensions.cs | 7 ++++ LLama/LLamaContext.cs | 8 ++-- LLama/LLamaStatelessExecutor.cs | 26 +++++------- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/LLama.Examples/NewVersion/StatelessModeExecute.cs b/LLama.Examples/NewVersion/StatelessModeExecute.cs index 7b75e373..cffb1e95 100644 --- a/LLama.Examples/NewVersion/StatelessModeExecute.cs +++ b/LLama.Examples/NewVersion/StatelessModeExecute.cs @@ -35,11 +35,49 @@ namespace LLama.Examples.NewVersion Console.ForegroundColor = ConsoleColor.White; Console.Write("Answer: "); prompt = $"Question: {prompt?.Trim()} Answer: "; - await foreach (var text in ex.InferAsync(prompt, inferenceParams)) + await foreach (var text in Spinner(ex.InferAsync(prompt, inferenceParams))) { Console.Write(text); } } } + + /// + /// Show a spinner while waiting for the next result + /// + /// + /// + private static async IAsyncEnumerable Spinner(IAsyncEnumerable source) + { + var enumerator = source.GetAsyncEnumerator(); + + var characters = new[] { '|', '/', '-', '\\' }; + + while (true) + { + var next = enumerator.MoveNextAsync(); + + var (Left, Top) = Console.GetCursorPosition(); + + // Keep showing the next spinner character while waiting for "MoveNextAsync" to finish + var count = 0; + while (!next.IsCompleted) + { + count = (count + 1) % characters.Length; + Console.SetCursorPosition(Left, Top); + Console.Write(characters[count]); + await Task.Delay(75); + } + + // Clear the spinner character + Console.SetCursorPosition(Left, Top); + Console.Write(" "); + Console.SetCursorPosition(Left, Top); + + if (!next.Result) + break; + yield return enumerator.Current; + } + } } } diff --git a/LLama/Extensions/IReadOnlyListExtensions.cs b/LLama/Extensions/IReadOnlyListExtensions.cs index 131a8852..4d1c6f09 100644 --- a/LLama/Extensions/IReadOnlyListExtensions.cs +++ b/LLama/Extensions/IReadOnlyListExtensions.cs @@ -68,6 +68,13 @@ namespace LLama.Extensions } } + internal static bool TokensEndsWithAnyString(this TTokens tokens, TQueries? queries, LLamaContext context) + where TTokens : IReadOnlyList + where TQueries : IReadOnlyList + { + return TokensEndsWithAnyString(tokens, queries, context.NativeHandle.ModelHandle, context.Encoding); + } + /// /// Check if the given set of tokens ends with any of the given strings /// diff --git a/LLama/LLamaContext.cs b/LLama/LLamaContext.cs index 3177c76b..2e0340e8 100644 --- a/LLama/LLamaContext.cs +++ b/LLama/LLamaContext.cs @@ -406,7 +406,7 @@ namespace LLama /// /// The updated `pastTokensCount`. /// - public int Eval(llama_token[] tokens, llama_token pastTokensCount) + public int Eval(llama_token[] tokens, int pastTokensCount) { return Eval(tokens.AsSpan(), pastTokensCount); } @@ -418,7 +418,7 @@ namespace LLama /// /// The updated `pastTokensCount`. /// - public int Eval(List tokens, llama_token pastTokensCount) + public int Eval(List tokens, int pastTokensCount) { #if NET5_0_OR_GREATER var span = CollectionsMarshal.AsSpan(tokens); @@ -448,7 +448,7 @@ namespace LLama /// /// The updated `pastTokensCount`. /// - public int Eval(ReadOnlyMemory tokens, llama_token pastTokensCount) + public int Eval(ReadOnlyMemory tokens, int pastTokensCount) { return Eval(tokens.Span, pastTokensCount); } @@ -460,7 +460,7 @@ namespace LLama /// /// The updated `pastTokensCount`. /// - public int Eval(ReadOnlySpan tokens, llama_token pastTokensCount) + public int Eval(ReadOnlySpan tokens, int pastTokensCount) { var total = tokens.Length; for(var i = 0; i < total; i += Params.BatchSize) diff --git a/LLama/LLamaStatelessExecutor.cs b/LLama/LLamaStatelessExecutor.cs index 5b1c4250..b09bd809 100644 --- a/LLama/LLamaStatelessExecutor.cs +++ b/LLama/LLamaStatelessExecutor.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Tasks; using LLama.Extensions; namespace LLama @@ -73,7 +74,6 @@ namespace LLama cancellationToken.ThrowIfCancellationRequested(); var antiprompts = inferenceParams?.AntiPrompts.ToArray() ?? Array.Empty(); - var n_past = 1; inferenceParams ??= new InferenceParams(); var lastTokens = new List(inferenceParams.RepeatLastTokensCount); @@ -81,12 +81,12 @@ namespace LLama lastTokens.Add(0); var tokens = Context.Tokenize(text).ToList(); - var n_prompt_tokens = tokens.Count; - Context.Eval(tokens, n_past); + await Task.Run(() => { Context.Eval(tokens, 1); }, cancellationToken) + .ConfigureAwait(false); lastTokens.AddRange(tokens); - n_past += n_prompt_tokens; + var n_past = 1 + tokens.Count; var mu = (float?)null; var max_tokens = inferenceParams.MaxTokens < 0 ? int.MaxValue : inferenceParams.MaxTokens; @@ -111,7 +111,8 @@ namespace LLama tokens.Clear(); tokens.Add(id); - if (EndsWithAntiprompt(lastTokens, antiprompts)) + // Check if any of the antiprompts have been generated + if (tokens.TokensEndsWithAnyString(antiprompts, Context)) break; // when run out of context @@ -126,19 +127,10 @@ namespace LLama tokens.AddRange(lastTokens.Skip(lastTokens.Count - n_left / 2).Take(n_left / 2)); } - n_past = Context.Eval(tokens, n_past); + // ReSharper disable once AccessToModifiedClosure (Justification: n_past is modified inside and outside the capture, but not concurrently) + n_past = await Task.Run(() => Context.Eval(tokens, n_past), cancellationToken) + .ConfigureAwait(false); } } - - /// - /// Check if the given tokens list ends with any of the antiprompts - /// - /// - /// - /// - private bool EndsWithAntiprompt(IReadOnlyList tokens, IReadOnlyList antiprompts) - { - return tokens.TokensEndsWithAnyString(antiprompts, Context.NativeHandle.ModelHandle, Context.Encoding); - } } } From d58fcbbd137ac7407a593587c3f5ffa4d38fe72a Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Sun, 24 Sep 2023 14:26:43 +0100 Subject: [PATCH 2/3] Fixed antiprompt checking --- LLama/LLamaStatelessExecutor.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LLama/LLamaStatelessExecutor.cs b/LLama/LLamaStatelessExecutor.cs index b09bd809..3ff755a0 100644 --- a/LLama/LLamaStatelessExecutor.cs +++ b/LLama/LLamaStatelessExecutor.cs @@ -104,15 +104,13 @@ namespace LLama inferenceParams.MirostatEta, inferenceParams.TopK, inferenceParams.TopP, inferenceParams.TfsZ, inferenceParams.TypicalP, inferenceParams.Grammar); lastTokens.Add(id); - - var response = Context.TokenToString(id); - yield return response; + yield return Context.TokenToString(id); tokens.Clear(); tokens.Add(id); // Check if any of the antiprompts have been generated - if (tokens.TokensEndsWithAnyString(antiprompts, Context)) + if (lastTokens.TokensEndsWithAnyString(antiprompts, Context)) break; // when run out of context From b7379b7124edec9742397730ecd678c45b315002 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Sun, 24 Sep 2023 19:24:52 +0100 Subject: [PATCH 3/3] Moved spinner out to an extension, so it can easily be used in other examples --- .../Extensions/IAsyncEnumerableExtensions.cs | 43 +++++++++++++++++++ .../NewVersion/StatelessModeExecute.cs | 41 +----------------- 2 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs diff --git a/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs b/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs new file mode 100644 index 00000000..b829a40a --- /dev/null +++ b/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs @@ -0,0 +1,43 @@ +namespace LLama.Examples.Extensions +{ + public static class IAsyncEnumerableExtensions + { + /// + /// Show a console spinner while waiting for the next result + /// + /// + /// + public static async IAsyncEnumerable Spinner(this IAsyncEnumerable source) + { + var enumerator = source.GetAsyncEnumerator(); + + var characters = new[] { '|', '/', '-', '\\' }; + + while (true) + { + var next = enumerator.MoveNextAsync(); + + var (Left, Top) = Console.GetCursorPosition(); + + // Keep showing the next spinner character while waiting for "MoveNextAsync" to finish + var count = 0; + while (!next.IsCompleted) + { + count = (count + 1) % characters.Length; + Console.SetCursorPosition(Left, Top); + Console.Write(characters[count]); + await Task.Delay(75); + } + + // Clear the spinner character + Console.SetCursorPosition(Left, Top); + Console.Write(" "); + Console.SetCursorPosition(Left, Top); + + if (!next.Result) + break; + yield return enumerator.Current; + } + } + } +} diff --git a/LLama.Examples/NewVersion/StatelessModeExecute.cs b/LLama.Examples/NewVersion/StatelessModeExecute.cs index cffb1e95..f4f030d9 100644 --- a/LLama.Examples/NewVersion/StatelessModeExecute.cs +++ b/LLama.Examples/NewVersion/StatelessModeExecute.cs @@ -1,4 +1,5 @@ using LLama.Common; +using LLama.Examples.Extensions; namespace LLama.Examples.NewVersion { @@ -35,49 +36,11 @@ namespace LLama.Examples.NewVersion Console.ForegroundColor = ConsoleColor.White; Console.Write("Answer: "); prompt = $"Question: {prompt?.Trim()} Answer: "; - await foreach (var text in Spinner(ex.InferAsync(prompt, inferenceParams))) + await foreach (var text in ex.InferAsync(prompt, inferenceParams).Spinner()) { Console.Write(text); } } } - - /// - /// Show a spinner while waiting for the next result - /// - /// - /// - private static async IAsyncEnumerable Spinner(IAsyncEnumerable source) - { - var enumerator = source.GetAsyncEnumerator(); - - var characters = new[] { '|', '/', '-', '\\' }; - - while (true) - { - var next = enumerator.MoveNextAsync(); - - var (Left, Top) = Console.GetCursorPosition(); - - // Keep showing the next spinner character while waiting for "MoveNextAsync" to finish - var count = 0; - while (!next.IsCompleted) - { - count = (count + 1) % characters.Length; - Console.SetCursorPosition(Left, Top); - Console.Write(characters[count]); - await Task.Delay(75); - } - - // Clear the spinner character - Console.SetCursorPosition(Left, Top); - Console.Write(" "); - Console.SetCursorPosition(Left, Top); - - if (!next.Result) - break; - yield return enumerator.Current; - } - } } }