diff --git a/LLama/Native/LLamaTokenDataArray.cs b/LLama/Native/LLamaTokenDataArray.cs index 4bc154f4..897cf8b8 100644 --- a/LLama/Native/LLamaTokenDataArray.cs +++ b/LLama/Native/LLamaTokenDataArray.cs @@ -145,15 +145,17 @@ namespace LLama.Native /// /// /// - public void RepetitionPenalty(SafeLLamaContextHandle context, Memory last_tokens, float penalty_repeat, float penalty_freq, float penalty_present) + public void RepetitionPenalty(SafeLLamaContextHandle context, ReadOnlySpan last_tokens, float penalty_repeat, float penalty_freq, float penalty_present) { unsafe { using (LLamaTokenDataArrayNative.Create(this, out var st)) - using (var last_tokens_handle = last_tokens.Pin()) { - NativeApi.llama_sample_repetition_penalties(context, ref st, (int*)last_tokens_handle.Pointer, (ulong)last_tokens.Length, penalty_repeat, penalty_freq, penalty_present); - sorted = st.sorted; + fixed (int* last_tokens_handle = last_tokens) + { + NativeApi.llama_sample_repetition_penalties(context, ref st, last_tokens_handle, (ulong)last_tokens.Length, penalty_repeat, penalty_freq, penalty_present); + sorted = st.sorted; + } } } } diff --git a/LLama/Sampling/ISamplingPipeline.cs b/LLama/Sampling/ISamplingPipeline.cs new file mode 100644 index 00000000..489f2c5a --- /dev/null +++ b/LLama/Sampling/ISamplingPipeline.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using LLama.Native; +using LLama.Sampling.Logits; +using LLama.Sampling.Selection; +using LLama.Sampling.Tokens; + +namespace LLama.Sampling; + +/// +/// Convert a span of logits into a single sampled token +/// +public interface ISamplingPipeline + : IDisposable +{ + /// + /// Sample a single token from the given logits + /// + /// + /// + /// + /// + int Sample(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens); + + /// + /// Reset all internal state of the sampling pipeline + /// + void Reset(); +} + +/// +/// Simple implementation of `ISamplingPipeline`, applies processors in order every time +/// +public sealed class BasicSamplingPipeline + : ISamplingPipeline +{ + /// + /// Logit processors to apply in this pipeline + /// + public IList LogitProcessors { get; } = new List(); + + /// + /// Token data processors to apply in this pipeline + /// + public IList TokenDataProcessors { get; } = new List(); + + /// + /// The selector to choose the final token + /// + public ITokenSelector Selector { get; set; } = new StandardSelection(); + + /// + public int Sample(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens) + { + // Modify raw logits + foreach (var logitProcessor in LogitProcessors) + logitProcessor.ProcessLogits(ctx, logits, lastTokens); + + // Convert logits into token candidates + var candidates_p = LLamaTokenDataArray.Create(logits); + + // Process token candidates + foreach (var tokenDataProcessor in TokenDataProcessors) + tokenDataProcessor.ProcessTokens(ctx, candidates_p, lastTokens); + + // Select a token + var token = Selector.Select(ctx, candidates_p, lastTokens); + + // Tell processors what was selected + foreach (var logitProcessor in LogitProcessors) + logitProcessor.AcceptToken(ctx, token); + foreach (var tokenDataProcessor in TokenDataProcessors) + tokenDataProcessor.AcceptToken(ctx, token); + + return token; + } + + /// + public void Reset() + { + foreach (var logitProcessor in LogitProcessors) + logitProcessor.Reset(); + foreach (var tokenDataProcessor in TokenDataProcessors) + tokenDataProcessor.Reset(); + + Selector.Reset(); + } + + /// + public void Dispose() + { + foreach (var logitProcessor in LogitProcessors) + logitProcessor.Dispose(); + foreach (var tokenDataProcessor in TokenDataProcessors) + tokenDataProcessor.Dispose(); + + Selector.Dispose(); + } +} \ No newline at end of file diff --git a/LLama/Sampling/Logits/ILogitProcessor.cs b/LLama/Sampling/Logits/ILogitProcessor.cs new file mode 100644 index 00000000..76968499 --- /dev/null +++ b/LLama/Sampling/Logits/ILogitProcessor.cs @@ -0,0 +1,34 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Logits; + +using llama_token = Int32; + +/// +/// Processes raw logits before sampling, applying penalties to certain tokens +/// +public interface ILogitProcessor + : IDisposable +{ + /// + /// Process raw logits, indexed by llama_token + /// + /// The context this is operating in + /// The token data array to process + /// The most recent tokens output + /// LLamaTokenDataArray, created from logits + void ProcessLogits(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens); + + /// + /// Inform this process when a token is accepted by the model + /// + /// + /// + void AcceptToken(SafeLLamaContextHandle ctx, int token); + + /// + /// Reset all internal sampling state + /// + void Reset(); +} \ No newline at end of file diff --git a/LLama/Sampling/Logits/LogitBias.cs b/LLama/Sampling/Logits/LogitBias.cs new file mode 100644 index 00000000..fc821508 --- /dev/null +++ b/LLama/Sampling/Logits/LogitBias.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using LLama.Native; + +namespace LLama.Sampling.Logits; + +/// +/// Add a bias directly to logit values +/// +public sealed class LogitBias + : ILogitProcessor +{ + /// + /// Biases to apply, token -> bias + /// + public IDictionary Biases { get; } = new Dictionary(); + + /// + public void ProcessLogits(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens) + { + foreach (var kvp in Biases) + logits[kvp.Key] += kvp.Value; + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Logits/SaveLoad.cs b/LLama/Sampling/Logits/SaveLoad.cs new file mode 100644 index 00000000..6f80aec4 --- /dev/null +++ b/LLama/Sampling/Logits/SaveLoad.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using LLama.Native; + +namespace LLama.Sampling.Logits; + +/// +/// Save certain logit values +/// +public sealed class SaveLogitValues + : ILogitProcessor +{ + private readonly Dictionary _saved = new(); + + /// + /// Logits to save + /// + public ISet Logits { get; } = new HashSet(); + + /// + /// Saved logit values + /// + public IReadOnlyDictionary Values => _saved; + + /// + public void ProcessLogits(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens) + { + _saved.Clear(); + foreach (var logit in Logits) + _saved[logit] = logits[logit]; + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + _saved.Clear(); + } + + /// + public void Dispose() + { + } + + /// + /// Get a logit processor that overwrite the logit values with the values saved here + /// + /// + public ILogitProcessor GetWriter() + { + return new LoadLogitValues(_saved); + } +} + +/// +/// Overwrite certain logit values +/// +public sealed class LoadLogitValues + : ILogitProcessor +{ + /// + /// Logits to overwrite, token -> logit + /// + public IDictionary Values { get; } + + /// + /// Create a new LoadLogitValues + /// + /// Source for values to overwrite + public LoadLogitValues(Dictionary? values = null) + { + Values = values ?? new Dictionary(); + } + + /// + public void ProcessLogits(SafeLLamaContextHandle ctx, Span logits, ReadOnlySpan lastTokens) + { + foreach (var logit in Values) + logits[logit.Key] = logit.Value; + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Selection/GreedySelection.cs b/LLama/Sampling/Selection/GreedySelection.cs new file mode 100644 index 00000000..30b72456 --- /dev/null +++ b/LLama/Sampling/Selection/GreedySelection.cs @@ -0,0 +1,27 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Selection; + +/// +/// Select the most likely token +/// +public sealed class GreedySelection + : ITokenSelector +{ + /// + public int Select(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens) + { + return candidates.SampleTokenGreedy(ctx); + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Selection/ITokenSelector.cs b/LLama/Sampling/Selection/ITokenSelector.cs new file mode 100644 index 00000000..c8915a92 --- /dev/null +++ b/LLama/Sampling/Selection/ITokenSelector.cs @@ -0,0 +1,25 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Selection; + +/// +/// Select a single token from a set of possibilities +/// +public interface ITokenSelector + : IDisposable +{ + /// + /// Select a single token + /// + /// + /// + /// + /// + int Select(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens); + + /// + /// Reset the state + /// + void Reset(); +} \ No newline at end of file diff --git a/LLama/Sampling/Selection/Mirostat2Selection.cs b/LLama/Sampling/Selection/Mirostat2Selection.cs new file mode 100644 index 00000000..cdc802c1 --- /dev/null +++ b/LLama/Sampling/Selection/Mirostat2Selection.cs @@ -0,0 +1,65 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Selection; + +/// +/// Select a token using Mirostat sampling. +/// Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. +/// +public sealed class Mirostat2Selection + : ITokenSelector +{ + private float _mu; + + /// + /// Current value of Mu, updated based on the difference between target surprise and actual surprise + /// + public float Mu + { + get => _mu; + set => _mu = value; + } + + /// + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. + /// A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text + /// + public float Tau { get; set; } + + /// + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. + /// A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// + public float Eta { get; set; } + + /// + /// Create a new Mirostat 2.0 sampler + /// + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. + /// A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. + /// A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + public Mirostat2Selection(float tau, float eta) + { + Tau = tau; + Eta = eta; + } + + /// + public int Select(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens) + { + return candidates.SampleTokenMirostat2(ctx, Tau, Eta, ref _mu); + } + + /// + public void Reset() + { + _mu = 2 * Tau; + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Selection/MirostatSelection.cs b/LLama/Sampling/Selection/MirostatSelection.cs new file mode 100644 index 00000000..5ec34a13 --- /dev/null +++ b/LLama/Sampling/Selection/MirostatSelection.cs @@ -0,0 +1,76 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Selection; + +/// +/// Select a token using Mirostat sampling. +/// Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. +/// +public sealed class MirostatSelection + : ITokenSelector +{ + private float _mu; + + /// + /// Current value of Mu, updated based on the difference between target surprise and actual surprise + /// + public float Mu + { + get => _mu; + set => _mu = value; + } + + /// + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. + /// A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text + /// + public float Tau { get; set; } + + /// + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. + /// A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// + public float Eta { get; set; } + + /// + /// The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn + /// helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects + /// the performance of the algorithm. + /// + public int M { get; set; } + + /// + /// Create a new Mirostat 2.0 sampler + /// + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. + /// A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. + /// A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn + /// helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects + /// the performance of the algorithm. + public MirostatSelection(float tau, float eta, int m = 100) + { + Tau = tau; + Eta = eta; + M = m; + } + + /// + public int Select(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens) + { + return candidates.SampleTokenMirostat(ctx, Tau, Eta, M, ref _mu); + } + + /// + public void Reset() + { + _mu = 2 * Tau; + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Selection/StandardSelection.cs b/LLama/Sampling/Selection/StandardSelection.cs new file mode 100644 index 00000000..3e3bd086 --- /dev/null +++ b/LLama/Sampling/Selection/StandardSelection.cs @@ -0,0 +1,27 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Selection; + +/// +/// Select from all possible tokens according to their probability +/// +public sealed class StandardSelection + : ITokenSelector +{ + /// + public int Select(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, ReadOnlySpan lastTokens) + { + return candidates.SampleToken(ctx); + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/GrammarSampling.cs b/LLama/Sampling/Tokens/GrammarSampling.cs new file mode 100644 index 00000000..b823a7f9 --- /dev/null +++ b/LLama/Sampling/Tokens/GrammarSampling.cs @@ -0,0 +1,59 @@ +using System; +using LLama.Grammars; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Apply a grammar to prevent sampling tokens which do not match the grammar +/// +public sealed class GrammarSampling + : ITokenDataProcessor +{ + private SafeLLamaGrammarHandle? _handle; + + /// + /// Grammar to use for sampling + /// + public Grammar? Grammar { get; set; } + + /// + /// Create a new + /// + /// + public GrammarSampling(Grammar grammar) + { + Grammar = grammar; + } + + /// + public void Reset() + { + _handle?.Dispose(); + _handle = null; + } + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + // Create a new grammar instance if necessary + _handle ??= Grammar?.CreateInstance(); + + // Apply it + if (_handle != null) + tokens.ApplyGrammar(ctx, _handle); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + _handle?.AcceptToken(ctx, token); + } + + /// + public void Dispose() + { + _handle?.Dispose(); + _handle = null; + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/ITokenDataProcessor.cs b/LLama/Sampling/Tokens/ITokenDataProcessor.cs new file mode 100644 index 00000000..e6679cc2 --- /dev/null +++ b/LLama/Sampling/Tokens/ITokenDataProcessor.cs @@ -0,0 +1,34 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +using llama_token = Int32; + +/// +/// Processes token logits before sampling, applying penalties to certain tokens +/// +public interface ITokenDataProcessor + : IDisposable +{ + /// + /// Process token logits in a LLamaTokenDataArray + /// + /// The context this is operating in + /// The token data array to process + /// The most recent tokens output + /// LLamaTokenDataArray, created from logits + void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens); + + /// + /// Inform this process when a token is accepted by the model + /// + /// + /// + void AcceptToken(SafeLLamaContextHandle ctx, int token); + + /// + /// Reset all internal sampling state + /// + void Reset(); +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/LocallyTypicalSampling.cs b/LLama/Sampling/Tokens/LocallyTypicalSampling.cs new file mode 100644 index 00000000..3f602c9a --- /dev/null +++ b/LLama/Sampling/Tokens/LocallyTypicalSampling.cs @@ -0,0 +1,42 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. +/// +public sealed class LocallyTypicalSampling + : ITokenDataProcessor +{ + /// + /// P value for locally typical sampling + /// + public float P { get; set; } + + /// + /// Minimum number of tokens to keep + /// + public ulong MinKeep { get; set; } = 1; + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.LocallyTypical(ctx, P, MinKeep); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/MinPSampling.cs b/LLama/Sampling/Tokens/MinPSampling.cs new file mode 100644 index 00000000..c3adf026 --- /dev/null +++ b/LLama/Sampling/Tokens/MinPSampling.cs @@ -0,0 +1,42 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 +/// +public sealed class MinPSampling + : ITokenDataProcessor +{ + /// + /// All tokens with probability greater than this will be kept + /// + public float P { get; set; } + + /// + /// Minimum number of tokens to keep + /// + public ulong MinKeep { get; set; } = 1; + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.MinP(ctx, P, MinKeep); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/RepetitionPenalty.cs b/LLama/Sampling/Tokens/RepetitionPenalty.cs new file mode 100644 index 00000000..3cfdbcd4 --- /dev/null +++ b/LLama/Sampling/Tokens/RepetitionPenalty.cs @@ -0,0 +1,77 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. +/// Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. +/// +public sealed class RepetitionPenalty + : ITokenDataProcessor +{ + private float _alphaFreq; + private float _alphaPresence; + + /// + /// Repetition penalty, as described in https://arxiv.org/abs/1909.05858 + /// + public float RepeatPenalty { get; set; } = 1.1f; + + /// + /// Frequency penalty as described by OpenAI: https://platform.openai.com/docs/api-reference/chat/create
+ /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text + /// so far, decreasing the model's likelihood to repeat the same line verbatim. + /// + public float AlphaFrequency + { + get => _alphaFreq; + set + { + if (value < -2) + throw new ArgumentOutOfRangeException(nameof(value), "AlphaFrequency must be greater than -2"); + if (value > 2) + throw new ArgumentOutOfRangeException(nameof(value), "AlphaFrequency must be less than 2"); + _alphaFreq = value; + } + } + + /// + /// Presence penalty as described by OpenAI: https://platform.openai.com/docs/api-reference/chat/create
+ /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the + /// text so far, increasing the model's likelihood to talk about new topics. + /// + public float AlphaPresence + { + get => _alphaPresence; + set + { + if (value < -2) + throw new ArgumentOutOfRangeException(nameof(value), "AlphaFrequency must be greater than -2"); + if (value > 2) + throw new ArgumentOutOfRangeException(nameof(value), "AlphaFrequency must be less than 2"); + _alphaPresence = value; + } + } + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.RepetitionPenalty(ctx, lastTokens, RepeatPenalty, AlphaFrequency, AlphaPresence); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/TailFreeSampling.cs b/LLama/Sampling/Tokens/TailFreeSampling.cs new file mode 100644 index 00000000..8e9fb2b5 --- /dev/null +++ b/LLama/Sampling/Tokens/TailFreeSampling.cs @@ -0,0 +1,42 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. +/// +public sealed class TailFreeSampling + : ITokenDataProcessor +{ + /// + /// Z value for tail free sampling + /// + public float Z { get; set; } + + /// + /// Minimum number of tokens to keep + /// + public ulong MinKeep { get; set; } = 1; + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.TailFree(ctx, Z, MinKeep); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/TemperatureSampling.cs b/LLama/Sampling/Tokens/TemperatureSampling.cs new file mode 100644 index 00000000..0186f275 --- /dev/null +++ b/LLama/Sampling/Tokens/TemperatureSampling.cs @@ -0,0 +1,38 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Sample with temperature. +/// As temperature increases, the prediction becomes more diverse but also vulnerable to hallucinations -- generating tokens that are sensible but not factual +/// +public sealed class TemperatureSampling + : ITokenDataProcessor +{ + /// + /// Temperature value to apply + /// + public float Temperature { get; set; } = 0.5f; + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.Temperature(ctx, Temperature); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/TopKSampling.cs b/LLama/Sampling/Tokens/TopKSampling.cs new file mode 100644 index 00000000..3f797c85 --- /dev/null +++ b/LLama/Sampling/Tokens/TopKSampling.cs @@ -0,0 +1,38 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Sample with TopK, removing all by the K most likely tokens. +/// Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +/// +public sealed class TopKSampling + : ITokenDataProcessor +{ + /// + /// Number of tokens to keep + /// + public int Count { get; set; } + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.TopK(ctx, Count); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file diff --git a/LLama/Sampling/Tokens/TopPSampling.cs b/LLama/Sampling/Tokens/TopPSampling.cs new file mode 100644 index 00000000..577ce3bc --- /dev/null +++ b/LLama/Sampling/Tokens/TopPSampling.cs @@ -0,0 +1,42 @@ +using System; +using LLama.Native; + +namespace LLama.Sampling.Tokens; + +/// +/// Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +/// +public sealed class TopPSampling + : ITokenDataProcessor +{ + /// + /// P valies for TopP + /// + public float P { get; set; } + + /// + /// Minimum number of tokens to keep + /// + public ulong MinKeep { get; set; } = 1; + + /// + public void ProcessTokens(SafeLLamaContextHandle ctx, LLamaTokenDataArray tokens, ReadOnlySpan lastTokens) + { + tokens.TopP(ctx, P, MinKeep); + } + + /// + public void AcceptToken(SafeLLamaContextHandle ctx, int token) + { + } + + /// + public void Reset() + { + } + + /// + public void Dispose() + { + } +} \ No newline at end of file