diff --git a/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs b/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs
new file mode 100644
index 00000000..b829a40a
--- /dev/null
+++ b/LLama.Examples/Extensions/IAsyncEnumerableExtensions.cs
@@ -0,0 +1,43 @@
+namespace LLama.Examples.Extensions
+{
+ public static class IAsyncEnumerableExtensions
+ {
+ ///
+ /// Show a console spinner while waiting for the next result
+ ///
+ ///
+ ///
+ public static async IAsyncEnumerable Spinner(this IAsyncEnumerable source)
+ {
+ var enumerator = source.GetAsyncEnumerator();
+
+ var characters = new[] { '|', '/', '-', '\\' };
+
+ while (true)
+ {
+ var next = enumerator.MoveNextAsync();
+
+ var (Left, Top) = Console.GetCursorPosition();
+
+ // Keep showing the next spinner character while waiting for "MoveNextAsync" to finish
+ var count = 0;
+ while (!next.IsCompleted)
+ {
+ count = (count + 1) % characters.Length;
+ Console.SetCursorPosition(Left, Top);
+ Console.Write(characters[count]);
+ await Task.Delay(75);
+ }
+
+ // Clear the spinner character
+ Console.SetCursorPosition(Left, Top);
+ Console.Write(" ");
+ Console.SetCursorPosition(Left, Top);
+
+ if (!next.Result)
+ break;
+ yield return enumerator.Current;
+ }
+ }
+ }
+}
diff --git a/LLama.Examples/NewVersion/StatelessModeExecute.cs b/LLama.Examples/NewVersion/StatelessModeExecute.cs
index 7b75e373..f4f030d9 100644
--- a/LLama.Examples/NewVersion/StatelessModeExecute.cs
+++ b/LLama.Examples/NewVersion/StatelessModeExecute.cs
@@ -1,4 +1,5 @@
using LLama.Common;
+using LLama.Examples.Extensions;
namespace LLama.Examples.NewVersion
{
@@ -35,7 +36,7 @@ namespace LLama.Examples.NewVersion
Console.ForegroundColor = ConsoleColor.White;
Console.Write("Answer: ");
prompt = $"Question: {prompt?.Trim()} Answer: ";
- await foreach (var text in ex.InferAsync(prompt, inferenceParams))
+ await foreach (var text in ex.InferAsync(prompt, inferenceParams).Spinner())
{
Console.Write(text);
}
diff --git a/LLama/Extensions/IReadOnlyListExtensions.cs b/LLama/Extensions/IReadOnlyListExtensions.cs
index 131a8852..4d1c6f09 100644
--- a/LLama/Extensions/IReadOnlyListExtensions.cs
+++ b/LLama/Extensions/IReadOnlyListExtensions.cs
@@ -68,6 +68,13 @@ namespace LLama.Extensions
}
}
+ internal static bool TokensEndsWithAnyString(this TTokens tokens, TQueries? queries, LLamaContext context)
+ where TTokens : IReadOnlyList
+ where TQueries : IReadOnlyList
+ {
+ return TokensEndsWithAnyString(tokens, queries, context.NativeHandle.ModelHandle, context.Encoding);
+ }
+
///
/// Check if the given set of tokens ends with any of the given strings
///
diff --git a/LLama/LLamaContext.cs b/LLama/LLamaContext.cs
index 3177c76b..2e0340e8 100644
--- a/LLama/LLamaContext.cs
+++ b/LLama/LLamaContext.cs
@@ -406,7 +406,7 @@ namespace LLama
///
/// The updated `pastTokensCount`.
///
- public int Eval(llama_token[] tokens, llama_token pastTokensCount)
+ public int Eval(llama_token[] tokens, int pastTokensCount)
{
return Eval(tokens.AsSpan(), pastTokensCount);
}
@@ -418,7 +418,7 @@ namespace LLama
///
/// The updated `pastTokensCount`.
///
- public int Eval(List tokens, llama_token pastTokensCount)
+ public int Eval(List tokens, int pastTokensCount)
{
#if NET5_0_OR_GREATER
var span = CollectionsMarshal.AsSpan(tokens);
@@ -448,7 +448,7 @@ namespace LLama
///
/// The updated `pastTokensCount`.
///
- public int Eval(ReadOnlyMemory tokens, llama_token pastTokensCount)
+ public int Eval(ReadOnlyMemory tokens, int pastTokensCount)
{
return Eval(tokens.Span, pastTokensCount);
}
@@ -460,7 +460,7 @@ namespace LLama
///
/// The updated `pastTokensCount`.
///
- public int Eval(ReadOnlySpan tokens, llama_token pastTokensCount)
+ public int Eval(ReadOnlySpan tokens, int pastTokensCount)
{
var total = tokens.Length;
for(var i = 0; i < total; i += Params.BatchSize)
diff --git a/LLama/LLamaStatelessExecutor.cs b/LLama/LLamaStatelessExecutor.cs
index 5b1c4250..3ff755a0 100644
--- a/LLama/LLamaStatelessExecutor.cs
+++ b/LLama/LLamaStatelessExecutor.cs
@@ -5,6 +5,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
+using System.Threading.Tasks;
using LLama.Extensions;
namespace LLama
@@ -73,7 +74,6 @@ namespace LLama
cancellationToken.ThrowIfCancellationRequested();
var antiprompts = inferenceParams?.AntiPrompts.ToArray() ?? Array.Empty();
- var n_past = 1;
inferenceParams ??= new InferenceParams();
var lastTokens = new List(inferenceParams.RepeatLastTokensCount);
@@ -81,12 +81,12 @@ namespace LLama
lastTokens.Add(0);
var tokens = Context.Tokenize(text).ToList();
- var n_prompt_tokens = tokens.Count;
- Context.Eval(tokens, n_past);
+ await Task.Run(() => { Context.Eval(tokens, 1); }, cancellationToken)
+ .ConfigureAwait(false);
lastTokens.AddRange(tokens);
- n_past += n_prompt_tokens;
+ var n_past = 1 + tokens.Count;
var mu = (float?)null;
var max_tokens = inferenceParams.MaxTokens < 0 ? int.MaxValue : inferenceParams.MaxTokens;
@@ -104,14 +104,13 @@ namespace LLama
inferenceParams.MirostatEta, inferenceParams.TopK, inferenceParams.TopP, inferenceParams.TfsZ, inferenceParams.TypicalP, inferenceParams.Grammar);
lastTokens.Add(id);
-
- var response = Context.TokenToString(id);
- yield return response;
+ yield return Context.TokenToString(id);
tokens.Clear();
tokens.Add(id);
- if (EndsWithAntiprompt(lastTokens, antiprompts))
+ // Check if any of the antiprompts have been generated
+ if (lastTokens.TokensEndsWithAnyString(antiprompts, Context))
break;
// when run out of context
@@ -126,19 +125,10 @@ namespace LLama
tokens.AddRange(lastTokens.Skip(lastTokens.Count - n_left / 2).Take(n_left / 2));
}
- n_past = Context.Eval(tokens, n_past);
+ // ReSharper disable once AccessToModifiedClosure (Justification: n_past is modified inside and outside the capture, but not concurrently)
+ n_past = await Task.Run(() => Context.Eval(tokens, n_past), cancellationToken)
+ .ConfigureAwait(false);
}
}
-
- ///
- /// Check if the given tokens list ends with any of the antiprompts
- ///
- ///
- ///
- ///
- private bool EndsWithAntiprompt(IReadOnlyList tokens, IReadOnlyList antiprompts)
- {
- return tokens.TokensEndsWithAnyString(antiprompts, Context.NativeHandle.ModelHandle, Context.Encoding);
- }
}
}