diff --git a/LLama/Abstractions/Params/ModelParams.cs b/LLama/Abstractions/Params/ModelParams.cs new file mode 100644 index 00000000..6d88d5e5 --- /dev/null +++ b/LLama/Abstractions/Params/ModelParams.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions.Params +{ + public class ModelParams + { + /// + /// Model context size (n_ctx) + /// + public int ContextSize { get; set; } = 512; + + /// + /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) + /// + public int GpuLayerCount { get; set; } = 20; + /// + /// Seed for the random number generator (seed) + /// + public int Seed { get; set; } = 1686349486; + /// + /// Use f16 instead of f32 for memory kv (memory_f16) + /// + public bool UseFp16Memory { get; set; } = true; + /// + /// Use mmap for faster loads (use_mmap) + /// + public bool UseMemorymap { get; set; } = true; + /// + /// Use mlock to keep model in memory (use_mlock) + /// + public bool UseMemoryLock { get; set; } = false; + /// + /// Compute perplexity over the prompt (perplexity) + /// + public bool Perplexity { get; set; } = false; + /// + /// Model path (model) + /// + public string ModelPath { get; set; } + /// + /// lora adapter path (lora_adapter) + /// + public string LoraAdapter { get; set; } = string.Empty; + /// + /// base model path for the lora adapter (lora_base) + /// + public string LoraBase { get; set; } = string.Empty; + /// + /// Number of threads (-1 = autodetect) (n_threads) + /// + public int Threads { get; set; } = Math.Max(Environment.ProcessorCount / 2, 1); + /// + /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) + /// + public int BatchSize { get; set; } = 512; + + /// + /// Whether to convert eos to newline during the inference. + /// + public bool ConvertEosToNewLine { get; set; } = false; + + /// + /// Whether to use embedding mode. (embedding) Note that if this is set to true, + /// The LLamaModel won't produce text response anymore. + /// + public bool EmbeddingMode { get; set; } = false; + + /// + /// + /// + /// The model path. + /// Model context size (n_ctx) + /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) + /// Seed for the random number generator (seed) + /// Whether to use f16 instead of f32 for memory kv (memory_f16) + /// Whether to use mmap for faster loads (use_mmap) + /// Whether to use mlock to keep model in memory (use_mlock) + /// Thether to compute perplexity over the prompt (perplexity) + /// Lora adapter path (lora_adapter) + /// Base model path for the lora adapter (lora_base) + /// Number of threads (-1 = autodetect) (n_threads) + /// Batch size for prompt processing (must be >=32 to use BLAS) (n_batch) + /// Whether to convert eos to newline during the inference. + /// Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore. + public ModelParams(string modelPath, int contextSize = 512, int gpuLayerCount = 20, + int seed = 1337, bool useFp16Memory = true, + bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false, + string loraAdapter = "", string loraBase = "", int threads = -1, int batchSize = 512, + bool convertEosToNewLine = false, bool embeddingMode = false) + { + ContextSize = contextSize; + GpuLayerCount = gpuLayerCount; + Seed = seed; + UseFp16Memory = useFp16Memory; + UseMemorymap = useMemorymap; + UseMemoryLock = useMemoryLock; + Perplexity = perplexity; + ModelPath = modelPath; + LoraAdapter = loraAdapter; + LoraBase = loraBase; + Threads = threads == -1 ? Math.Max(Environment.ProcessorCount / 2, 1) : threads; + BatchSize = batchSize; + ConvertEosToNewLine = convertEosToNewLine; + EmbeddingMode = embeddingMode; + } + } +} diff --git a/LLama/Abstractions/Params/SessionParams.cs b/LLama/Abstractions/Params/SessionParams.cs new file mode 100644 index 00000000..41a28c21 --- /dev/null +++ b/LLama/Abstractions/Params/SessionParams.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Abstractions.Params +{ + using llama_token = Int32; + public class SessionParams + { + /// + /// number of tokens to keep from initial prompt + /// + public int TokensToKeep { get; set; } = 0; + /// + /// how many new tokens to predict (n_predict), set to -1 to inifinitely generate response + /// until it complete. + /// + public int ResponseTokensCount { get; set; } = -1; + /// + /// logit bias for specific tokens + /// + public Dictionary? LogitBias { get; set; } = null; + /// + /// path to file for saving/loading model eval state + /// + public string PathSession { get; set; } = string.Empty; + /// + /// string to suffix user inputs with + /// + public string InputSuffix { get; set; } = string.Empty; + /// + /// string to prefix user inputs with + /// + public string InputPrefix { get; set; } = string.Empty; + /// + /// 0 or lower to use vocab size + /// + public int TopK { get; set; } = 40; + /// + /// 1.0 = disabled + /// + public float TopP { get; set; } = 0.95f; + /// + /// 1.0 = disabled + /// + public float TfsZ { get; set; } = 1.0f; + /// + /// 1.0 = disabled + /// + public float TypicalP { get; set; } = 1.0f; + /// + /// 1.0 = disabled + /// + public float Temperature { get; set; } = 0.8f; + /// + /// 1.0 = disabled + /// + public float RepeatPenalty { get; set; } = 1.1f; + /// + /// last n tokens to penalize (0 = disable penalty, -1 = context size) (repeat_last_n) + /// + public int RepeatLastTokensCount { get; set; } = 64; + /// + /// frequency penalty coefficient + /// 0.0 = disabled + /// + public float FrequencyPenalty { get; set; } = .0f; + /// + /// presence penalty coefficient + /// 0.0 = disabled + /// + public float PresencePenalty { get; set; } = .0f; + /// + /// Mirostat uses tokens instead of words. + /// algorithm described in the paper https://arxiv.org/abs/2007.14966. + /// 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 + /// + public MiroStateType Mirostat { get; set; } = MiroStateType.Disable; + /// + /// target entropy + /// + public float MirostatTau { get; set; } = 5.0f; + /// + /// learning rate + /// + public float MirostatEta { get; set; } = 0.1f; + /// + /// consider newlines as a repeatable token (penalize_nl) + /// + public bool PenalizeNL { get; set; } = true; + } + + public enum MiroStateType + { + Disable = 0, + MiroState = 1, + MiroState2 = 2 + } +} diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs index 501d0977..559deb48 100644 --- a/LLama/ChatSession.cs +++ b/LLama/ChatSession.cs @@ -1,53 +1,102 @@ -using LLama.Types; +using LLama.Abstractions.Params; +using LLama.Common; +using LLama.Exceptions; +using LLama.Native; using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Reflection; using System.Text; namespace LLama { - public class ChatSession where T: IChatModel - { - IChatModel _model; - List History { get; } = new List(); - - public ChatSession(T model) - { - _model = model; - } - - public IEnumerable Chat(string text, string? prompt = null, string encoding = "UTF-8") - { - History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Human, text), DateTime.Now)); - string totalResponse = ""; - foreach(var response in _model.Chat(text, prompt, encoding)) - { - totalResponse += response; - yield return response; - } - History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Assistant, totalResponse), DateTime.Now)); - } - - public ChatSession WithPrompt(string prompt, string encoding = "UTF-8") - { - _model.InitChatPrompt(prompt, encoding); - return this; - } - - public ChatSession WithPromptFile(string promptFilename, string encoding = "UTF-8") - { - return WithPrompt(File.ReadAllText(promptFilename), encoding); - } - - /// - /// Set the keyword to split the return value of chat AI. - /// - /// - /// - public ChatSession WithAntiprompt(string[] antiprompt) - { - _model.InitChatAntiprompt(antiprompt); - return this; - } - } -} + using llama_token = Int32; + //public class ChatHistoryEntry + //{ + // public string Role { get; set; } + // public string Text { get; set; } + //} + + //public class ChatMetadata + //{ + // public string Prompt { get; set; } = "Prompt"; + // public IEnumerable? AntiPrompts { get; set; } = null; + // public string User { get; set; } = "User"; + // public string Assistant { get; set; } = "Assistant"; + + // public ChatMetadata SetPrompt(string v) + // { + // Prompt = v; + // return this; + // } + + // public ChatMetadata SetUserName(string v) + // { + // User = v; + // return this; + // } + + // public ChatMetadata SetAssistantName(string v) + // { + // Assistant = v; + // return this; + // } + + // public ChatMetadata WithPromptFromFile(string filename) + // { + // Prompt = System.IO.File.ReadAllText(filename); + // return this; + // } + //} + + //public class ChatSession + //{ + // private LLamaModel _model; + // private ChatMetadata _metadata; + + // public List ChatHistory { get; } = new(); + // public ChatSession(LLamaModel model, ChatMetadata? metadata = null) + // { + // _model = model; + // if (metadata == null) metadata = new ChatMetadata(); + // _metadata = metadata; + + // if (_metadata.Prompt != "") + // { + // ChatHistory.Add(new ChatHistoryEntry() { Role = "", Text = _metadata.Prompt }); + // } + // } + + // string _formatChatHistory(List history) + // { + // StringBuilder sb = new(); + // foreach (var entry in history) + // { + // if (entry.Role == "") + // { + // sb.Append($"{entry.Text}\n"); + // continue; + // } + // sb.Append($"{entry.Role}: {entry.Text}\n"); + // } + // sb.Append($"{_metadata.Assistant}: "); + // return sb.ToString(); + // } + + // public IEnumerable Chat(string text) + // { + // ChatHistory.Add(new ChatHistoryEntry() { Role = "User", Text = text }); + // string totalResponse = ""; + // //foreach (var response in _model.GenerateResult(_formatChatHistory(ChatHistory), null, _metadata.AntiPrompts)) + // //{ + // // totalResponse += response; + // // yield return response; + // //} + // ChatHistory.Add(new ChatHistoryEntry() { Role = "Assistant", Text = totalResponse }); + // } + //} + + + +} \ No newline at end of file diff --git a/LLama/Common/FixedQuene.cs b/LLama/Common/FixedQuene.cs new file mode 100644 index 00000000..ba8a24a7 --- /dev/null +++ b/LLama/Common/FixedQuene.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace LLama.Common +{ + /// + /// A queue with fixed storage size. + /// Currently it's only a naive implementation and needs to be further optimized in the future. + /// + public class FixedSizeQuene: IEnumerable + { + int _maxSize; + List _storage; + + public int Count => _storage.Count; + public FixedSizeQuene(int size) + { + _maxSize = size; + _storage = new(); + } + + public FixedSizeQuene FillWith(T value) + { + for(int i = 0; i < Count; i++) + { + _storage[i] = value; + } + return this; + } + + /// + /// Enquene an element. + /// + /// + public void Enqueue(T item) + { + _storage.Add(item); + if(_storage.Count >= _maxSize) + { + _storage.RemoveAt(0); + } + } + + public T[] ToArray() + { + return _storage.ToArray(); + } + + public IEnumerator GetEnumerator() + { + return _storage.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/LLama/ILLamaExecutor.cs b/LLama/ILLamaExecutor.cs new file mode 100644 index 00000000..4e773637 --- /dev/null +++ b/LLama/ILLamaExecutor.cs @@ -0,0 +1,12 @@ +using LLama.Abstractions.Params; +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama +{ + public interface ILLamaExecutor + { + IEnumerable Infer(string text, SessionParams? sessionParams = null, IEnumerable? antiprompts = null); + } +} diff --git a/LLama/LLamaExecutorBase.cs b/LLama/LLamaExecutorBase.cs new file mode 100644 index 00000000..61d9a192 --- /dev/null +++ b/LLama/LLamaExecutorBase.cs @@ -0,0 +1,111 @@ +using LLama.Abstractions.Params; +using LLama.Common; +using LLama.Exceptions; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace LLama +{ + using llama_token = Int32; + public abstract class LLamaExecutorBase: ILLamaExecutor + { + protected LLamaModel _model; + protected int _pastTokensCount; // n_past + protected int _consumedTokensCount; // n_consume + protected int _n_session_consumed; + protected int _n_matching_session_tokens; + protected string _pathSession; + protected List _embeds = new(); // embd + protected List _embed_inps = new(); + protected List _session_tokens = new(); + protected FixedSizeQuene _last_n_tokens; + protected LLamaExecutorBase(LLamaModel model) + { + _model = model; + _pastTokensCount = 0; + _consumedTokensCount = 0; + _n_session_consumed = 0; + _embeds = new(); + _embed_inps = new(); + _last_n_tokens = new FixedSizeQuene(_model.ContextSize).FillWith(0); + } + + public unsafe LLamaExecutorBase WithSessionFile(string filename) + { + _pathSession = filename; + if (string.IsNullOrEmpty(filename)) + { + throw new ArgumentNullException("File name cannot be empty."); + } + if (File.Exists(filename)) + { + llama_token[] session_tokens = new llama_token[_model.ContextSize]; + ulong n_token_count_out = 0; + if (!NativeApi.llama_load_session_file(_model.NativeHandle, _pathSession, session_tokens, (ulong)_model.ContextSize, &n_token_count_out)) + { + throw new RuntimeError($"Failed to load session file {_pathSession}"); + } + _session_tokens = session_tokens.Take((int)n_token_count_out).ToList(); + } + return this; + } + + public void SaveSessionFile(string filename) + { + var session_token_array = _session_tokens.ToArray(); + NativeApi.llama_save_session_file(_model.NativeHandle, filename, session_token_array, (ulong)session_token_array.Length); + } + + protected virtual void HandleRunOutOfContext(int tokensToKeep) + { + // if we run out of context: + // - take the tokensToKeep first tokens from the original prompt (via n_past) + // - take half of the last (n_ctx - tokensToKeep) tokens and recompute the logits in batches + int n_left = _pastTokensCount - tokensToKeep; + + _pastTokensCount = Math.Max(1, tokensToKeep); + + // insert n_left/2 tokens at the start of embed from last_n_tokens + _embeds.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embeds.Count).Skip(_model.ContextSize - n_left / 2 - _embeds.Count)); + + // stop saving session if we run out of context + _pathSession = string.Empty; + } + + protected virtual void TryReuseMathingPrefix() + { + if (_n_session_consumed < _session_tokens.Count) + { + int i = 0; + for (; i < _embeds.Count; i++) + { + if (_embeds[i] != _session_tokens[_n_session_consumed]) + { + _session_tokens = _session_tokens.Take(_n_session_consumed).ToList(); + break; + } + + _pastTokensCount++; + _n_session_consumed++; + + if (_n_session_consumed >= _session_tokens.Count) + { + i++; + break; + } + } + + if (i > 0) + { + _embeds.RemoveRange(0, i); + } + } + } + + public abstract IEnumerable Infer(string text, SessionParams? sessionParams = null, IEnumerable? antiprompts = null); + } +} diff --git a/LLama/LLamaInstructExecutor.cs b/LLama/LLamaInstructExecutor.cs new file mode 100644 index 00000000..220c2182 --- /dev/null +++ b/LLama/LLamaInstructExecutor.cs @@ -0,0 +1,200 @@ +using LLama.Abstractions.Params; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace LLama +{ + using llama_token = Int32; + public class LLamaInstructExecutor : LLamaExecutorBase + { + bool _prompt_run = true; + readonly IEnumerable _llama_token_newline; + readonly IEnumerable _inp_pfx; + readonly IEnumerable _inp_sfx; + public LLamaInstructExecutor(LLamaModel model, string inputPrefix = "\n\n### Instruction:\n\n", + string inputSuffix = "\n\n### Response:\n\n") : base(model) + { + _llama_token_newline = Utils.Tokenize(_model.NativeHandle, "\n", false, _model.Encoding); + _inp_pfx = _model.Tokenize(inputPrefix, true); + _inp_sfx = _model.Tokenize(inputSuffix, false); + } + + /// + /// process the text and return the tokens consumed. + /// + /// + /// + /// + /// + /// + protected virtual int ProcessTextBeforeInfer(string text, SessionParams sessionParams) + { + if (text.Length > 1) + { + if (!text.EndsWith("\n")) + { + text += "\n"; + } + _consumedTokensCount = _embed_inps.Count; + _embed_inps.AddRange(_inp_pfx); + + var line_inp = _model.Tokenize(text, false); + _embed_inps.AddRange(line_inp); + + _embed_inps.AddRange(_inp_sfx); + + return line_inp.Count(); + } + else + { + return 0; + } + } + + public override IEnumerable Infer(string text, SessionParams? sessionParams = null, IEnumerable? antiprompts = null) + { + if (sessionParams is null) + { + sessionParams = new SessionParams(); + } + // if n_remain < 0, the response will be generated endlessly. + int n_remain = sessionParams.ResponseTokensCount; + bool return_value = false; + bool wait_for_input = false; + bool need_to_save_session = !string.IsNullOrEmpty(_pathSession) && _n_matching_session_tokens < _embed_inps.Count; + + if (_prompt_run) + { + // When running the first input (prompt) in inteactive mode, we should specially process it. + text = " " + text; + _embed_inps = _model.Tokenize(text, true).ToList(); + } + else + { + n_remain -= ProcessTextBeforeInfer(text, sessionParams); + } + + while (n_remain != 0 || _prompt_run) + { + if (_embeds.Count > 0) + { + _prompt_run = false; + if (_pastTokensCount + _embeds.Count > _model.ContextSize) + { + HandleRunOutOfContext(sessionParams.TokensToKeep); + } + + TryReuseMathingPrefix(); + _pastTokensCount = _model.Eval(_embeds.ToArray(), _pastTokensCount); + + if (_embeds.Count > 0 && !string.IsNullOrEmpty(_pathSession)) + { + _session_tokens.AddRange(_embeds); + _n_session_consumed = _session_tokens.Count; + } + } + + _embeds.Clear(); + + if (_embed_inps.Count <= _consumedTokensCount && !wait_for_input) + { + var temp = sessionParams.Temperature; + var top_k = sessionParams.TopK <= 0 ? NativeApi.llama_n_vocab(_model.NativeHandle) : sessionParams.TopK; + var top_p = sessionParams.TopK; + var tfs_z = sessionParams.TfsZ; + var typical_p = sessionParams.TypicalP; + var repeat_last_n = sessionParams.RepeatLastTokensCount < 0 ? _model.ContextSize : sessionParams.RepeatLastTokensCount; + var repeat_penalty = sessionParams.RepeatPenalty; + var alpha_presence = sessionParams.PresencePenalty; + var alpha_frequency = sessionParams.FrequencyPenalty; + var mirostat = sessionParams.Mirostat; + var mirostat_tau = sessionParams.MirostatTau; + var mirostat_eta = sessionParams.MirostatEta; + var penalize_nl = sessionParams.PenalizeNL; + + // optionally save the session on first sample (for faster prompt loading next time) + if (!string.IsNullOrEmpty(_pathSession) && need_to_save_session) + { + need_to_save_session = false; + SaveSessionFile(_pathSession); + } + + var tokenDataArray = _model.ApplyPenalty(_last_n_tokens, sessionParams.LogitBias, repeat_last_n, + repeat_penalty, alpha_frequency, alpha_presence, penalize_nl); + + var id = _model.Sample(tokenDataArray, temp, mirostat, mirostat_tau, mirostat_eta, top_k, top_p, + tfs_z, typical_p); + + _last_n_tokens.Enqueue(id); + + _embeds.Add(id); + + n_remain--; + return_value = true; + } + else + { + while (_embed_inps.Count > _consumedTokensCount) + { + _embeds.Add(_embed_inps[_consumedTokensCount]); + _last_n_tokens.Enqueue(_embed_inps[_consumedTokensCount]); + _consumedTokensCount++; + if (_embeds.Count >= _model.Params.BatchSize) + { + break; + } + } + } + + if (return_value) + { + foreach (var item in _model.GenerateResult(_embeds)) + { + yield return item; + } + } + + if (_embed_inps.Count <= _consumedTokensCount) + { + if (antiprompts is not null && antiprompts.Count() > 0) + { + string last_output = ""; + foreach (var id in _last_n_tokens) + { + last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding); + } + + foreach (var antiprompt in antiprompts) + { + if (last_output.EndsWith(antiprompt)) + { + wait_for_input = true; + break; + } + } + } + + if (_pastTokensCount > 0 && wait_for_input) + { + yield return "\n> "; + break; + } + } + + if (_embeds.Count > 0 && _embeds.Last() == NativeApi.llama_token_eos()) + { + wait_for_input = true; + } + + if (n_remain <= 0 && sessionParams.ResponseTokensCount != -1) + { + n_remain = sessionParams.ResponseTokensCount; + wait_for_input = true; + } + } + } + } +} diff --git a/LLama/LLamaInteractExecutor.cs b/LLama/LLamaInteractExecutor.cs new file mode 100644 index 00000000..e2671467 --- /dev/null +++ b/LLama/LLamaInteractExecutor.cs @@ -0,0 +1,203 @@ +using LLama.Abstractions.Params; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace LLama +{ + using llama_token = Int32; + public class LLamaInteractExecutor : LLamaExecutorBase + { + bool _prompt_run = true; + readonly IEnumerable _llama_token_newline; + readonly IEnumerable _inp_pfx; + readonly IEnumerable _inp_sfx; + public LLamaInteractExecutor(LLamaModel model) : base(model) + { + _llama_token_newline = Utils.Tokenize(_model.NativeHandle, "\n", false, _model.Encoding); + _inp_pfx = _model.Tokenize("\n\n### Instruction:\n\n", true); + _inp_sfx = _model.Tokenize("\n\n### Response:\n\n", false); + } + + /// + /// process the text and return the tokens consumed. + /// + /// + /// + /// + /// + /// + protected virtual int ProcessTextBeforeInfer(string text, SessionParams sessionParams) + { + if (text.Length > 1) + { + if (!text.EndsWith("\n")) + { + text += "\n"; + } + var line_inp = _model.Tokenize(text, false); + _embed_inps.AddRange(line_inp); + return line_inp.Count(); + } + else + { + return 0; + } + } + + public override IEnumerable Infer(string text, SessionParams? sessionParams = null, IEnumerable? antiprompts = null) + { + if (sessionParams is null) + { + sessionParams = new SessionParams(); + } + // if n_remain < 0, the response will be generated endlessly. + int n_remain = sessionParams.ResponseTokensCount; + bool return_value = false; + bool wait_for_input = false; + bool need_to_save_session = !string.IsNullOrEmpty(_pathSession) && _n_matching_session_tokens < _embed_inps.Count; + + if (_prompt_run) + { + // When running the first input (prompt) in inteactive mode, we should specially process it. + text = " " + text; + _embed_inps = _model.Tokenize(text, true).ToList(); + } + else + { + n_remain -= ProcessTextBeforeInfer(text, sessionParams); + } + + while (n_remain != 0 && !wait_for_input || _prompt_run) + { + if (_embeds.Count > 0) + { + _prompt_run = false; + if (_pastTokensCount + _embeds.Count > _model.ContextSize) + { + HandleRunOutOfContext(sessionParams.TokensToKeep); + } + + TryReuseMathingPrefix(); + _pastTokensCount = _model.Eval(_embeds.ToArray(), _pastTokensCount); + + if (_embeds.Count > 0 && !string.IsNullOrEmpty(_pathSession)) + { + _session_tokens.AddRange(_embeds); + _n_session_consumed = _session_tokens.Count; + } + } + + _embeds.Clear(); + + if (_embed_inps.Count <= _consumedTokensCount && !wait_for_input) + { + var temp = sessionParams.Temperature; + var top_k = sessionParams.TopK <= 0 ? NativeApi.llama_n_vocab(_model.NativeHandle) : sessionParams.TopK; + var top_p = sessionParams.TopK; + var tfs_z = sessionParams.TfsZ; + var typical_p = sessionParams.TypicalP; + var repeat_last_n = sessionParams.RepeatLastTokensCount < 0 ? _model.ContextSize : sessionParams.RepeatLastTokensCount; + var repeat_penalty = sessionParams.RepeatPenalty; + var alpha_presence = sessionParams.PresencePenalty; + var alpha_frequency = sessionParams.FrequencyPenalty; + var mirostat = sessionParams.Mirostat; + var mirostat_tau = sessionParams.MirostatTau; + var mirostat_eta = sessionParams.MirostatEta; + var penalize_nl = sessionParams.PenalizeNL; + + // optionally save the session on first sample (for faster prompt loading next time) + if (!string.IsNullOrEmpty(_pathSession) && need_to_save_session) + { + need_to_save_session = false; + SaveSessionFile(_pathSession); + } + + var tokenDataArray = _model.ApplyPenalty(_last_n_tokens, sessionParams.LogitBias, repeat_last_n, + repeat_penalty, alpha_frequency, alpha_presence, penalize_nl); + + var id = _model.Sample(tokenDataArray, temp, mirostat, mirostat_tau, mirostat_eta, top_k, top_p, + tfs_z, typical_p); + + _last_n_tokens.Enqueue(id); + + if (id == NativeApi.llama_token_eos()) + { + id = _llama_token_newline.First(); + if (antiprompts is not null && antiprompts.Count() > 0) + { + var first_antiprompt = _model.Tokenize(antiprompts.First(), false); + _embed_inps.AddRange(first_antiprompt); + } + } + + _embeds.Add(id); + + n_remain--; + return_value = true; + } + else + { + while (_embed_inps.Count > _consumedTokensCount) + { + _embeds.Add(_embed_inps[_consumedTokensCount]); + _last_n_tokens.Enqueue(_embed_inps[_consumedTokensCount]); + _consumedTokensCount++; + if (_embeds.Count >= _model.Params.BatchSize) + { + break; + } + } + } + + if (return_value) + { + foreach (var item in _model.GenerateResult(_embeds)) + { + yield return item; + } + } + + if (_embed_inps.Count <= _consumedTokensCount) + { + if (antiprompts is not null && antiprompts.Count() > 0) + { + string last_output = ""; + foreach (var id in _last_n_tokens) + { + last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding); + } + + foreach (var antiprompt in antiprompts) + { + if (last_output.EndsWith(antiprompt)) + { + wait_for_input = true; + break; + } + } + } + + if (_pastTokensCount > 0 && wait_for_input) + { + break; + } + } + + if (_embeds.Count > 0 && _embeds.Last() == NativeApi.llama_token_eos()) + { + yield return " [end of text]\n"; + break; + } + + if (n_remain <= 0 && sessionParams.ResponseTokensCount != -1) + { + n_remain = sessionParams.ResponseTokensCount; + wait_for_input = true; + } + } + } + } +} diff --git a/LLama/LLamaModel.cs b/LLama/LLamaModel.cs index 6ca9ebd4..6c7219e5 100644 --- a/LLama/LLamaModel.cs +++ b/LLama/LLamaModel.cs @@ -1,804 +1,188 @@ -using LLama.Exceptions; +using LLama.Abstractions.Params; +using LLama.Exceptions; +using LLama.Native; +using LLama.Old; using LLama.Types; using LLama.Extensions; -using LLama.Native; using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; using System.Text; +using System.Threading; namespace LLama { using llama_token = Int32; - public class LLamaModel : IChatModel, IDisposable + public class LLamaModel { - LLamaParams _params; + // TODO: expose more properties. + LLamaLogger _logger; + Encoding _encoding; SafeLLamaContextHandle _ctx; - string _path_session; - List _session_tokens; - List _embed_inp; - int _n_ctx; - List _inp_pfx; - List _inp_sfx; - List _llama_token_newline; - List _last_n_tokens; - bool _is_interacting; - bool _is_antiprompt; - bool _input_echo; - bool _verbose; - - // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session - // if we loaded a session with at least 75% similarity. It's currently just used to speed up the - // initial prompt so it doesn't need to be an exact match. - bool _need_to_save_session; - int _n_past; - int _n_remain; - int _n_consumed; - int _n_session_consumed; - List _embed; - - public string Name { get; set; } - public bool Verbose - { - get - { - return _verbose; - } - set - { - _verbose = value; - } - } + public int ContextSize { get; } + public ModelParams Params { get; set; } public SafeLLamaContextHandle NativeHandle => _ctx; + public Encoding Encoding => _encoding; - /// - /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will - /// load 20 layers to gpu by default. - /// - /// The model file path. - /// The model name. - /// Whether to print details when running the model. - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, - int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1, - Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, - float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, - int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f, - int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "", - string path_session = "", string input_prefix = "", string input_suffix = "", - List antiprompt = null, string lora_adapter = "", string lora_base = "", - bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false, - bool embedding = false, bool interactive_first = false, bool prompt_cache_all = false, bool instruct = false, bool penalize_nl = true, - bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false, - bool verbose_prompt = false, string encoding = "UTF-8") : this(new LLamaParams(seed: seed, - n_threads: n_threads, - n_predict: n_predict, - n_ctx: n_ctx, - n_batch: n_batch, - n_keep: n_keep, - n_gpu_layers: n_gpu_layers, - logit_bias: logit_bias, - top_k: top_k, - top_p: top_p, - tfs_z: tfs_z, - typical_p: typical_p, - temp: temp, - repeat_penalty: repeat_penalty, - repeat_last_n: repeat_last_n, - frequency_penalty: frequency_penalty, - presence_penalty: presence_penalty, - mirostat: mirostat, - mirostat_tau: mirostat_tau, - mirostat_eta: mirostat_eta, - model: model_path, - prompt: prompt, - path_session: path_session, - input_prefix: input_prefix, - input_suffix: input_suffix, - antiprompt: antiprompt, - lora_adapter: lora_adapter, - lora_base: lora_base, - memory_f16: memory_f16, - random_prompt: random_prompt, - use_color: use_color, - interactive: interactive, - embedding: embedding, - interactive_first: interactive_first, - prompt_cache_all: prompt_cache_all, - instruct: instruct, - penalize_nl: penalize_nl, - perplexity: perplexity, - use_mmap: use_mmap, - use_mlock: use_mlock, - mem_test: mem_test, - verbose_prompt: verbose_prompt), - model_name, verbose, encoding) + public void Dispose() { - + _ctx.Dispose(); } - /// - /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will - /// load 20 layers to gpu by default. - /// - /// The LLamaModel params - /// Model name - /// Whether to output the detailed info. - /// - /// - public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8") + public LLamaModel(ModelParams Params, string encoding = "UTF-8") { - Name = name; - _params = @params; - _verbose = verbose; - _ctx = Utils.llama_init_from_gpt_params(ref _params); - - // Add a space in front of the first character to match OG llama tokenizer behavior - _session_tokens = new List(); - - _path_session = @params.path_session; - if (!string.IsNullOrEmpty(_path_session)) - { - if (verbose) - { - LLamaLogger.Default.Info($"Attempting to load saved session from '{_path_session}'"); - } - - if (!File.Exists(_path_session)) - { - LLamaLogger.Default.Warn("Session file does not exist, will create."); - } - - llama_token[] session_tokens = new llama_token[@params.n_ctx]; - ulong n_token_count_out = 0; - if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out)) - { - throw new RuntimeError($"Failed to load session file {_path_session}"); - } - _session_tokens = session_tokens.Take((int)n_token_count_out).ToList(); - if (verbose) - { - LLamaLogger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens"); - } - } - - _n_ctx = NativeApi.llama_n_ctx(_ctx); - - WithPrompt(_params.prompt); - - // prefix & suffix for instruct mode - _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true, encoding); - _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false, encoding); - - // in instruct mode, we inject a prefix and a suffix to each input by the user - if (_params.instruct) - { - _params.interactive_first = true; - _params.antiprompt.Add("### Instruction:\n\n"); - } - - // enable interactive mode if reverse prompt or interactive start is specified - if (_params.interactive_first) - { - _params.interactive = true; - } - - // determine newline token - _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false, encoding); - - if (_params.verbose_prompt) - { - LLamaLogger.Default.Info("\n"); - LLamaLogger.Default.Info($"prompt: '{_params.prompt}'"); - LLamaLogger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}"); - for (int i = 0; i < _embed_inp.Count; i++) - { - LLamaLogger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'"); - } - if (_params.n_keep > 0) - { - LLamaLogger.Default.Info($"static prompt based on n_keep: '"); - for (int i = 0; i < _params.n_keep; i++) - { - LLamaLogger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}"); - } - LLamaLogger.Default.Info("\n"); - } - LLamaLogger.Default.Info("\n"); - } - - if (_params.interactive && verbose) - { - LLamaLogger.Default.Info("interactive mode on."); - } - if (verbose) - { - LLamaLogger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " + - $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + - $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + - $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + - $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); - LLamaLogger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " + - $"n_keep = {_params.n_keep}"); - LLamaLogger.Default.Info("\n"); - } - - _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList(); - - if (_params.interactive) - { - if (verbose) - { - LLamaLogger.Default.Info("== Running in interactive mode. =="); - } - _is_interacting = _params.interactive_first; - } - - _is_antiprompt = false; - _input_echo = false; - _n_past = 0; - _n_remain = _params.n_predict; - _n_consumed = 0; - _n_session_consumed = 0; - _embed = new List(); + _logger = LLamaLogger.Default; + this.Params = Params; + _encoding = Encoding.GetEncoding(encoding); + _logger.Info($"Initializing LLama model with params: {this.Params}"); + _ctx = Utils.InitLLamaContextFromModelParams(this.Params); + ContextSize = NativeApi.llama_n_ctx(_ctx); } /// - /// Apply a prompt to the model. + /// Tokenize a string. /// - /// - /// + /// + /// Whether to add a bos to the text. /// - /// - public LLamaModel WithPrompt(string prompt, string encoding = "UTF-8") + public IEnumerable Tokenize(string text, bool addBos = true) { - _params.prompt = prompt.Insert(0, " "); - _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true, encoding); - - if (_embed_inp.Count > _n_ctx - 4) - { - throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); - } - - ulong n_matching_session_tokens = 0; - if (_session_tokens.Count > 0) - { - foreach (var id in _session_tokens) - { - if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens]) - { - break; - } - n_matching_session_tokens++; - } - if (n_matching_session_tokens >= (ulong)_embed_inp.Count) - { - LLamaLogger.Default.Info("Session file has exact match for prompt!"); - } - else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2)) - { - LLamaLogger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " + - $"/ {_embed_inp.Count} tokens); will mostly be reevaluated."); - } - else - { - LLamaLogger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " + - $"tokens of prompt."); - } - } - // number of tokens to keep when resetting context - if (_params.n_keep < 0 || _params.n_keep > (int)_embed_inp.Count || _params.instruct) - { - _params.n_keep = _embed_inp.Count; - } - if (_embed_inp.Count > _n_ctx - 4) - { - throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); - } - _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4); - - return this; + // TODO: reconsider whether to convert to array here. + return Utils.Tokenize(_ctx, text, addBos, _encoding); } /// - /// Apply the prompt file to the model. + /// Detokenize the tokens to text. /// - /// + /// /// - public LLamaModel WithPromptFile(string promptFileName) + public string DeTokenize(IEnumerable tokens) { - return WithPrompt(File.ReadAllText(promptFileName)); + StringBuilder sb = new(); + foreach(var token in tokens) + { + sb.Append(Utils.PtrToString(NativeApi.llama_token_to_str(_ctx, token), _encoding)); + } + return sb.ToString(); } - private void ProcessTextBeforeInfer(string text, string encoding) + public llama_token Sample(LLamaTokenDataArray candidates, float temperature = 0.8f, MiroStateType mirostat = MiroStateType.Disable, + float mirostatTau = 5.0f, float mirostatEta = 0.1f, int topK = 40, float topP = 0.95f, float tfsZ = 1.0f, float typicalP = 1.0f) { - if (!string.IsNullOrEmpty(_params.input_prefix)) + llama_token id = 0; + if (temperature <= 0) { - text = _params.input_prefix + text; + // Greedy sampling + id = SamplingApi.llama_sample_token_greedy(_ctx, candidates); } - //if (!text.EndsWith("\n")) - //{ - // text += "\n"; - //} - if (text.Length > 1) + else { - // append input suffix if any - if (!string.IsNullOrEmpty(_params.input_suffix)) + if (mirostat == MiroStateType.MiroState) { - text += _params.input_suffix; - //yield return _params.input_suffix; + float mirostat_mu = 2.0f * mirostatTau; + const int mirostat_m = 100; + SamplingApi.llama_sample_temperature(_ctx, candidates, temperature); + id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates, mirostatTau, mirostatEta, mirostat_m, ref mirostat_mu); } - - // instruct mode: insert instruction prefix - if (_params.instruct && !_is_antiprompt) + else if (mirostat == MiroStateType.MiroState2) { - _n_consumed = _embed_inp.Count; - _embed_inp.AddRange(_inp_pfx); + float mirostat_mu = 2.0f * mirostatTau; + SamplingApi.llama_sample_temperature(_ctx, candidates, temperature); + id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates, mirostatTau, mirostatEta, ref mirostat_mu); } - - var line_inp = Utils.llama_tokenize(_ctx, text, false, encoding); - _embed_inp.AddRange(line_inp); - - // instruct mode: insert response suffix - if (_params.instruct) + else { - _embed_inp.AddRange(_inp_sfx); + // Temperature sampling + SamplingApi.llama_sample_top_k(_ctx, candidates, topK, 1); + SamplingApi.llama_sample_tail_free(_ctx, candidates, tfsZ, 1); + SamplingApi.llama_sample_typical(_ctx, candidates, typicalP, 1); + SamplingApi.llama_sample_top_p(_ctx, candidates, topP, 1); + SamplingApi.llama_sample_temperature(_ctx, candidates, temperature); + id = SamplingApi.llama_sample_token(_ctx, candidates); } - - _n_remain -= line_inp.Count; } + return id; } - public void InitChatPrompt(string prompt, string encoding = "UTF-8") + public LLamaTokenDataArray ApplyPenalty(IEnumerable lastTokens, Dictionary? logitBias = null, + int repeatLastTokensCount = 64, float repeatPenalty = 1.1f, float alphaFrequency = .0f, float alphaPresence = .0f, + bool penalizeNL = true) { - WithPrompt(prompt); - } + var n_vocab = NativeApi.llama_n_vocab(_ctx); + var logits = Utils.GetLogits(_ctx, n_vocab); - public void InitChatAntiprompt(string[] antiprompt) - { - _params.antiprompt = antiprompt.ToList(); - } - - /// - /// Chat with the LLaMa model under interactive mode. - /// - /// - /// - /// - /// - /// - public IEnumerable Chat(string text, string? prompt = null, string encoding = "UTF-8") - { - if (!_params.interactive) - { - throw new ArgumentException("The chat API could be only used under interactive model."); - } - _input_echo = false; - if (!string.IsNullOrEmpty(prompt)) + // Apply params.logit_bias map + if(logitBias is not null) { - WithPrompt(prompt); + foreach (var (key, value) in logitBias) + { + logits[key] += value; + } } - return Call(text, encoding); - } - - /// - /// Save the state to specified path. - /// - /// - public void SaveState(string filename) - { - var stateSize = NativeApi.llama_get_state_size(_ctx); - byte[] stateMemory = new byte[stateSize]; - NativeApi.llama_copy_state_data(_ctx, stateMemory); - File.WriteAllBytes(filename, stateMemory); - } - /// - /// Load the state from specified path. - /// - /// - /// Whether to clear previous footprints of this model. - /// - public void LoadState(string filename, bool clearPreviousEmbed = true) - { - var stateMemory = File.ReadAllBytes(filename); - int stateSize = (int)NativeApi.llama_get_state_size(_ctx); - if (stateMemory.Length != stateSize) + var candidates = new List(); + candidates.Capacity = n_vocab; + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { - throw new RuntimeError("Failed to validate state size."); + candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f)); } - NativeApi.llama_set_state_data(_ctx, stateMemory); - if (clearPreviousEmbed) - { - WithPrompt(_params.prompt); - } - } + LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false); - /// - /// Tokenize a string. - /// - /// The utf-8 encoded string to tokenize. - /// A list of tokens. - /// If the tokenization failed. - public List Tokenize(string text, string encoding = "UTF-8") - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - var n_ctx = NativeApi.llama_n_ctx(_ctx); - var tokens = new llama_token[n_ctx]; - var n_tokens = NativeApi.llama_tokenize(_ctx, text, Encoding.GetEncoding(encoding), tokens, n_ctx, true); - if (n_tokens < 0) + // Apply penalties + float nl_logit = logits[NativeApi.llama_token_nl()]; + int lastTokensCount = lastTokens.Count(); + var last_n_repeat = Math.Min(Math.Min(lastTokensCount, repeatLastTokensCount), ContextSize); + SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p, + lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(), + (ulong)last_n_repeat, repeatPenalty); + SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p, + lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(), + (ulong)last_n_repeat, alphaFrequency, alphaPresence); + if (!penalizeNL) { - throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}"); + logits[NativeApi.llama_token_nl()] = nl_logit; } - return tokens.Take(n_tokens).ToList(); - } - /// - /// Detokenize a list of tokens. - /// - /// The list of tokens to detokenize. - /// The detokenized string. - public string DeTokenize(IEnumerable tokens) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - string output = ""; - foreach (var token in tokens) - { - output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)); - } - return output; + return candidates_p; } /// - /// Call the model to run inference. + /// /// - /// - /// - /// + /// + /// + /// The updated `pastTokensCount`. /// - public IEnumerable Call(string text, string encoding = "UTF-8") + public llama_token Eval(llama_token[] tokens, llama_token pastTokensCount) { - _is_antiprompt = false; - if(_n_past > 0) - { - _is_interacting = false; - } - if (_is_interacting) - { - if (_verbose) - { - LLamaLogger.Default.Warn("In interacting when calling the model, automatically changed it."); - } - _is_interacting = false; - } - ProcessTextBeforeInfer(text, encoding); - - while ((_n_remain != 0 || _params.interactive) && !_is_interacting) + int total = tokens.Length; + for(int i = 0; i < total; i += Params.BatchSize) { - if (_embed.Count > 0) + int n_eval = total - i; + if(n_eval > Params.BatchSize) { - // infinite text generation via context swapping - // if we run out of context: - // - take the n_keep first tokens from the original prompt (via n_past) - // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches - if (_n_past + _embed.Count > _n_ctx) - { - int n_left = _n_past - _params.n_keep; - - _n_past = Math.Max(1, _params.n_keep); - - // insert n_left/2 tokens at the start of embed from last_n_tokens - _embed.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embed.Count).Skip(_n_ctx - n_left / 2 - _embed.Count)); - - // stop saving session if we run out of context - _path_session = ""; - } - - // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) - // REVIEW - if (_n_session_consumed < _session_tokens.Count) - { - int i = 0; - for (; i < _embed.Count; i++) - { - if (_embed[i] != _session_tokens[_n_session_consumed]) - { - _session_tokens = _session_tokens.Take(_n_session_consumed).ToList(); - break; - } - - _n_past++; - _n_session_consumed++; - - if (_n_session_consumed >= _session_tokens.Count) - { - i++; - break; - } - } - - if (i > 0) - { - _embed.RemoveRange(0, i); - } - } - - // evaluate tokens in batches - // embed is typically prepared beforehand to fit within a batch, but not always - for (int i = 0; i < _embed.Count; i += _params.n_batch) - { - int n_eval = _embed.Count - i; - - if (n_eval > _params.n_batch) - { - n_eval = _params.n_batch; - } - - var array = _embed.Skip(i).ToArray(); - if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0) - { - LLamaLogger.Default.Error($"Failed to eval."); - throw new RuntimeError("Failed to eval."); - } - - _n_past += n_eval; - } - - if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session)) - { - _session_tokens.AddRange(_embed); - _n_session_consumed = _session_tokens.Count; - } + n_eval = Params.BatchSize; } - _embed.Clear(); - - if (_embed_inp.Count <= _n_consumed && !_is_interacting) + if(Utils.Eval(_ctx, tokens, i, n_eval, pastTokensCount, Params.Threads) != 0) { - var temp = _params.temp; - var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k; - var top_p = _params.top_p; - var tfs_z = _params.tfs_z; - var typical_p = _params.typical_p; - var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n; - var repeat_penalty = _params.repeat_penalty; - var alpha_presence = _params.presence_penalty; - var alpha_frequency = _params.frequency_penalty; - var mirostat = _params.mirostat; - var mirostat_tau = _params.mirostat_tau; - var mirostat_eta = _params.mirostat_eta; - var penalize_nl = _params.penalize_nl; - - // optionally save the session on first sample (for faster prompt loading next time) - if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session) - { - _need_to_save_session = false; - NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count); - } - - llama_token id = 0; - - { - var n_vocab = NativeApi.llama_n_vocab(_ctx); - var logits = Utils.llama_get_logits(_ctx, n_vocab); - - // Apply params.logit_bias map - foreach (var (key, value) in _params.logit_bias) - { - logits[key] += value; - } - - var candidates = new List(); - candidates.Capacity = n_vocab; - for (llama_token token_id = 0; token_id < n_vocab; token_id++) - { - candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f)); - } - - LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false); - - // Apply penalties - float nl_logit = logits[NativeApi.llama_token_nl()]; - var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx); - SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p, - _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(), - (ulong)last_n_repeat, repeat_penalty); - SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p, - _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(), - (ulong)last_n_repeat, alpha_frequency, alpha_presence); - if (!penalize_nl) - { - logits[NativeApi.llama_token_nl()] = nl_logit; - } - - if (temp <= 0) - { - // Greedy sampling - id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p); - } - else - { - if (mirostat == 1) - { - float mirostat_mu = 2.0f * mirostat_tau; - const int mirostat_m = 100; - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, ref mirostat_mu); - } - else if (mirostat == 2) - { - float mirostat_mu = 2.0f * mirostat_tau; - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, ref mirostat_mu); - } - else - { - // Temperature sampling - SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1); - SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1); - SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1); - SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1); - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token(_ctx, candidates_p); - } - } - - _last_n_tokens.RemoveAt(0); - _last_n_tokens.Add(id); - } - - // replace end of text token with newline token when in interactive mode - if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct) - { - id = _llama_token_newline[0]; - if (_params.antiprompt.Count != 0) - { - // tokenize and inject first reverse prompt - var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false, encoding); - _embed_inp.AddRange(first_antiprompt); - } - } - - // add it to the context - _embed.Add(id); - - // echo this to console - _input_echo = true; - - // decrement remaining sampling budget - _n_remain--; - } - else - { - while (_embed_inp.Count > _n_consumed) - { - _embed.Add(_embed_inp[_n_consumed]); - _last_n_tokens.RemoveAt(0); - _last_n_tokens.Add(_embed_inp[_n_consumed]); - _n_consumed++; - if (_embed.Count >= _params.n_batch) - { - break; - } - } - } - - if (_input_echo && !_is_interacting) - { - foreach (var id in _embed) - { - var res = Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); - yield return res; - } - } - - if (_params.interactive && _embed_inp.Count <= _n_consumed) - { - if (_params.antiprompt.Count > 0) - { - string last_output = ""; - foreach (var id in _last_n_tokens) - { - last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); - } - - _is_antiprompt = false; - foreach (var antiprompt in _params.antiprompt) - { - if (last_output.EndsWith(antiprompt)) - { - _is_interacting = true; - _is_antiprompt = true; - break; - } - } - } - - if (_n_past > 0 && _is_interacting) - { - if (_params.instruct) - { - yield return "\n> "; - } - _input_echo = false; - break; - } - - if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos()) - { - if (_params.instruct) - { - _is_interacting = true; - } - else - { - LLamaLogger.Default.Info(" [end of text]"); - } - } - - if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1) - { - _n_remain = _params.n_predict; - _is_interacting = true; - } + _logger.Error($"Failed to eval."); + throw new RuntimeError("Failed to eval."); } - } - if (!string.IsNullOrEmpty(_path_session) && _params.prompt_cache_all) - { - LLamaLogger.Default.Info($"saving final output to session file {_path_session}"); - var session_token_array = _session_tokens.ToArray(); - NativeApi.llama_save_session_file(_ctx, _path_session, session_token_array, (ulong)session_token_array.Length); + pastTokensCount += n_eval; } + return pastTokensCount; } - public void Dispose() + // TODO: add comment + internal IEnumerable GenerateResult(IEnumerable ids) { - _ctx.Dispose(); + foreach(var id in ids) + { + yield return Utils.TokenToString(id, _ctx, _encoding); + } } } } diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 4b1f31c9..3d24cfe8 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -70,4 +70,10 @@ + + + + + + diff --git a/LLama/Native/LLamaTokenData.cs b/LLama/Native/LLamaTokenData.cs index cc877e80..a5ffda59 100644 --- a/LLama/Native/LLamaTokenData.cs +++ b/LLama/Native/LLamaTokenData.cs @@ -6,7 +6,7 @@ using System.Text; namespace LLama.Native { [StructLayout(LayoutKind.Sequential)] - internal struct LLamaTokenData + public struct LLamaTokenData { /// /// token id diff --git a/LLama/Native/LLamaTokenDataArray.cs b/LLama/Native/LLamaTokenDataArray.cs index 78d71e53..65e09564 100644 --- a/LLama/Native/LLamaTokenDataArray.cs +++ b/LLama/Native/LLamaTokenDataArray.cs @@ -7,7 +7,7 @@ using System.Text; namespace LLama.Native { [StructLayout(LayoutKind.Sequential)] - internal struct LLamaTokenDataArray + public struct LLamaTokenDataArray { public Memory data; public ulong size; @@ -23,7 +23,7 @@ namespace LLama.Native } [StructLayout(LayoutKind.Sequential)] - internal struct LLamaTokenDataArrayNative + public struct LLamaTokenDataArrayNative { public IntPtr data; public ulong size; diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 07ac4efe..63a07adf 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -164,6 +164,9 @@ namespace LLama.Native [DllImport(libraryName)] public static extern int llama_eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int n_tokens, int n_past, int n_threads); + [DllImport(libraryName, EntryPoint = "llama_eval")] + public static extern int llama_eval_with_pointer(SafeLLamaContextHandle ctx, llama_token* tokens, int n_tokens, int n_past, int n_threads); + /// /// Convert the provided text into tokens. /// The tokens pointer must be large enough to hold the resulting tokens. diff --git a/LLama/Old/ChatSession.cs b/LLama/Old/ChatSession.cs new file mode 100644 index 00000000..c1b4ca2d --- /dev/null +++ b/LLama/Old/ChatSession.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace LLama.Old +{ + public class ChatSession where T : IChatModel + { + IChatModel _model; + List History { get; } = new List(); + + public ChatSession(T model) + { + _model = model; + } + + public IEnumerable Chat(string text, string? prompt = null, string encoding = "UTF-8") + { + History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Human, text), DateTime.Now)); + string totalResponse = ""; + foreach (var response in _model.Chat(text, prompt, encoding)) + { + totalResponse += response; + yield return response; + } + History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Assistant, totalResponse), DateTime.Now)); + } + + public ChatSession WithPrompt(string prompt, string encoding = "UTF-8") + { + _model.InitChatPrompt(prompt, encoding); + return this; + } + + public ChatSession WithPromptFile(string promptFilename, string encoding = "UTF-8") + { + return WithPrompt(File.ReadAllText(promptFilename), encoding); + } + + /// + /// Set the keyword to split the return value of chat AI. + /// + /// + /// + public ChatSession WithAntiprompt(string[] antiprompt) + { + _model.InitChatAntiprompt(antiprompt); + return this; + } + } +} diff --git a/LLama/IChatModel.cs b/LLama/Old/IChatModel.cs similarity index 96% rename from LLama/IChatModel.cs rename to LLama/Old/IChatModel.cs index 71dad3fd..3324292a 100644 --- a/LLama/IChatModel.cs +++ b/LLama/Old/IChatModel.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace LLama +namespace LLama.Old { public interface IChatModel { diff --git a/LLama/LLamaEmbedder.cs b/LLama/Old/LLamaEmbedder.cs similarity index 93% rename from LLama/LLamaEmbedder.cs rename to LLama/Old/LLamaEmbedder.cs index 7652496b..9de2f58e 100644 --- a/LLama/LLamaEmbedder.cs +++ b/LLama/Old/LLamaEmbedder.cs @@ -4,9 +4,9 @@ using System.Collections.Generic; using System.Text; using LLama.Exceptions; -namespace LLama +namespace LLama.Old { - public class LLamaEmbedder: IDisposable + public class LLamaEmbedder : IDisposable { SafeLLamaContextHandle _ctx; @@ -27,7 +27,7 @@ namespace LLama public unsafe float[] GetEmbeddings(string text, int n_thread = -1, bool add_bos = true, string encoding = "UTF-8") { - if(n_thread == -1) + if (n_thread == -1) { n_thread = Math.Max(Environment.ProcessorCount / 2, 1); } @@ -51,7 +51,7 @@ namespace LLama int n_embed = NativeApi.llama_n_embd(_ctx); var embeddings = NativeApi.llama_get_embeddings(_ctx); - if(embeddings == null) + if (embeddings == null) { return new float[0]; } diff --git a/LLama/Old/LLamaModel.cs b/LLama/Old/LLamaModel.cs new file mode 100644 index 00000000..0c9488bb --- /dev/null +++ b/LLama/Old/LLamaModel.cs @@ -0,0 +1,804 @@ +using LLama.Exceptions; +using LLama.Types; +using LLama.Extensions; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; + +namespace LLama.Old +{ + using llama_token = Int32; + public class LLamaModel : IChatModel, IDisposable + { + LLamaParams _params; + SafeLLamaContextHandle _ctx; + string _path_session; + List _session_tokens; + List _embed_inp; + int _n_ctx; + List _inp_pfx; + List _inp_sfx; + List _llama_token_newline; + List _last_n_tokens; + bool _is_interacting; + bool _is_antiprompt; + bool _input_echo; + bool _verbose; + + // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session + // if we loaded a session with at least 75% similarity. It's currently just used to speed up the + // initial prompt so it doesn't need to be an exact match. + bool _need_to_save_session; + int _n_past; + int _n_remain; + int _n_consumed; + int _n_session_consumed; + List _embed; + + public string Name { get; set; } + public bool Verbose + { + get + { + return _verbose; + } + set + { + _verbose = value; + } + } + public SafeLLamaContextHandle NativeHandle => _ctx; + + /// + /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will + /// load 20 layers to gpu by default. + /// + /// The model file path. + /// The model name. + /// Whether to print details when running the model. + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, + int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1, + Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, + float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, + int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f, + int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "", + string path_session = "", string input_prefix = "", string input_suffix = "", + List antiprompt = null, string lora_adapter = "", string lora_base = "", + bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false, + bool embedding = false, bool interactive_first = false, bool prompt_cache_all = false, bool instruct = false, bool penalize_nl = true, + bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false, + bool verbose_prompt = false, string encoding = "UTF-8") : this(new LLamaParams(seed: seed, + n_threads: n_threads, + n_predict: n_predict, + n_ctx: n_ctx, + n_batch: n_batch, + n_keep: n_keep, + n_gpu_layers: n_gpu_layers, + logit_bias: logit_bias, + top_k: top_k, + top_p: top_p, + tfs_z: tfs_z, + typical_p: typical_p, + temp: temp, + repeat_penalty: repeat_penalty, + repeat_last_n: repeat_last_n, + frequency_penalty: frequency_penalty, + presence_penalty: presence_penalty, + mirostat: mirostat, + mirostat_tau: mirostat_tau, + mirostat_eta: mirostat_eta, + model: model_path, + prompt: prompt, + path_session: path_session, + input_prefix: input_prefix, + input_suffix: input_suffix, + antiprompt: antiprompt, + lora_adapter: lora_adapter, + lora_base: lora_base, + memory_f16: memory_f16, + random_prompt: random_prompt, + use_color: use_color, + interactive: interactive, + embedding: embedding, + interactive_first: interactive_first, + prompt_cache_all: prompt_cache_all, + instruct: instruct, + penalize_nl: penalize_nl, + perplexity: perplexity, + use_mmap: use_mmap, + use_mlock: use_mlock, + mem_test: mem_test, + verbose_prompt: verbose_prompt), + model_name, verbose, encoding) + { + + } + + /// + /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will + /// load 20 layers to gpu by default. + /// + /// The LLamaModel params + /// Model name + /// Whether to output the detailed info. + /// + /// + public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8") + { + Name = name; + _params = @params; + _verbose = verbose; + _ctx = Utils.llama_init_from_gpt_params(ref _params); + + // Add a space in front of the first character to match OG llama tokenizer behavior + _session_tokens = new List(); + + _path_session = @params.path_session; + if (!string.IsNullOrEmpty(_path_session)) + { + if (verbose) + { + LLamaLogger.Default.Info($"Attempting to load saved session from '{_path_session}'"); + } + + if (!File.Exists(_path_session)) + { + LLamaLogger.Default.Warn("Session file does not exist, will create."); + } + + llama_token[] session_tokens = new llama_token[@params.n_ctx]; + ulong n_token_count_out = 0; + if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out)) + { + throw new RuntimeError($"Failed to load session file {_path_session}"); + } + _session_tokens = session_tokens.Take((int)n_token_count_out).ToList(); + if (verbose) + { + LLamaLogger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens"); + } + } + + _n_ctx = NativeApi.llama_n_ctx(_ctx); + + WithPrompt(_params.prompt); + + // prefix & suffix for instruct mode + _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true, encoding); + _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false, encoding); + + // in instruct mode, we inject a prefix and a suffix to each input by the user + if (_params.instruct) + { + _params.interactive_first = true; + _params.antiprompt.Add("### Instruction:\n\n"); + } + + // enable interactive mode if reverse prompt or interactive start is specified + if (_params.interactive_first) + { + _params.interactive = true; + } + + // determine newline token + _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false, encoding); + + if (_params.verbose_prompt) + { + LLamaLogger.Default.Info("\n"); + LLamaLogger.Default.Info($"prompt: '{_params.prompt}'"); + LLamaLogger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}"); + for (int i = 0; i < _embed_inp.Count; i++) + { + LLamaLogger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'"); + } + if (_params.n_keep > 0) + { + LLamaLogger.Default.Info($"static prompt based on n_keep: '"); + for (int i = 0; i < _params.n_keep; i++) + { + LLamaLogger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}"); + } + LLamaLogger.Default.Info("\n"); + } + LLamaLogger.Default.Info("\n"); + } + + if (_params.interactive && verbose) + { + LLamaLogger.Default.Info("interactive mode on."); + } + if (verbose) + { + LLamaLogger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " + + $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + + $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + + $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + + $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); + LLamaLogger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " + + $"n_keep = {_params.n_keep}"); + LLamaLogger.Default.Info("\n"); + } + + _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList(); + + if (_params.interactive) + { + if (verbose) + { + LLamaLogger.Default.Info("== Running in interactive mode. =="); + } + _is_interacting = _params.interactive_first; + } + + _is_antiprompt = false; + _input_echo = false; + _n_past = 0; + _n_remain = _params.n_predict; + _n_consumed = 0; + _n_session_consumed = 0; + _embed = new List(); + } + + /// + /// Apply a prompt to the model. + /// + /// + /// + /// + /// + public LLamaModel WithPrompt(string prompt, string encoding = "UTF-8") + { + _params.prompt = prompt.Insert(0, " "); + _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true, encoding); + + if (_embed_inp.Count > _n_ctx - 4) + { + throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); + } + + ulong n_matching_session_tokens = 0; + if (_session_tokens.Count > 0) + { + foreach (var id in _session_tokens) + { + if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens]) + { + break; + } + n_matching_session_tokens++; + } + if (n_matching_session_tokens >= (ulong)_embed_inp.Count) + { + LLamaLogger.Default.Info("Session file has exact match for prompt!"); + } + else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2)) + { + LLamaLogger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " + + $"/ {_embed_inp.Count} tokens); will mostly be reevaluated."); + } + else + { + LLamaLogger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " + + $"tokens of prompt."); + } + } + // number of tokens to keep when resetting context + if (_params.n_keep < 0 || _params.n_keep > _embed_inp.Count || _params.instruct) + { + _params.n_keep = _embed_inp.Count; + } + if (_embed_inp.Count > _n_ctx - 4) + { + throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); + } + _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4); + + return this; + } + + /// + /// Apply the prompt file to the model. + /// + /// + /// + public LLamaModel WithPromptFile(string promptFileName) + { + return WithPrompt(File.ReadAllText(promptFileName)); + } + + private void ProcessTextBeforeInfer(string text, string encoding) + { + if (!string.IsNullOrEmpty(_params.input_prefix)) + { + text = _params.input_prefix + text; + } + //if (!text.EndsWith("\n")) + //{ + // text += "\n"; + //} + if (text.Length > 1) + { + // append input suffix if any + if (!string.IsNullOrEmpty(_params.input_suffix)) + { + text += _params.input_suffix; + //yield return _params.input_suffix; + } + + // instruct mode: insert instruction prefix + if (_params.instruct && !_is_antiprompt) + { + _n_consumed = _embed_inp.Count; + _embed_inp.AddRange(_inp_pfx); + } + + var line_inp = Utils.llama_tokenize(_ctx, text, false, encoding); + _embed_inp.AddRange(line_inp); + + // instruct mode: insert response suffix + if (_params.instruct) + { + _embed_inp.AddRange(_inp_sfx); + } + + _n_remain -= line_inp.Count; + } + } + + public void InitChatPrompt(string prompt, string encoding = "UTF-8") + { + WithPrompt(prompt); + } + + public void InitChatAntiprompt(string[] antiprompt) + { + _params.antiprompt = antiprompt.ToList(); + } + + /// + /// Chat with the LLaMa model under interactive mode. + /// + /// + /// + /// + /// + /// + public IEnumerable Chat(string text, string? prompt = null, string encoding = "UTF-8") + { + if (!_params.interactive) + { + throw new ArgumentException("The chat API could be only used under interactive model."); + } + _input_echo = false; + if (!string.IsNullOrEmpty(prompt)) + { + WithPrompt(prompt); + } + return Call(text, encoding); + } + + /// + /// Save the state to specified path. + /// + /// + public void SaveState(string filename) + { + var stateSize = NativeApi.llama_get_state_size(_ctx); + byte[] stateMemory = new byte[stateSize]; + NativeApi.llama_copy_state_data(_ctx, stateMemory); + File.WriteAllBytes(filename, stateMemory); + } + + /// + /// Load the state from specified path. + /// + /// + /// Whether to clear previous footprints of this model. + /// + public void LoadState(string filename, bool clearPreviousEmbed = true) + { + var stateMemory = File.ReadAllBytes(filename); + int stateSize = (int)NativeApi.llama_get_state_size(_ctx); + if (stateMemory.Length != stateSize) + { + throw new RuntimeError("Failed to validate state size."); + } + NativeApi.llama_set_state_data(_ctx, stateMemory); + + if (clearPreviousEmbed) + { + WithPrompt(_params.prompt); + } + } + + /// + /// Tokenize a string. + /// + /// The utf-8 encoded string to tokenize. + /// A list of tokens. + /// If the tokenization failed. + public List Tokenize(string text, string encoding = "UTF-8") + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var n_ctx = NativeApi.llama_n_ctx(_ctx); + var tokens = new llama_token[n_ctx]; + var n_tokens = NativeApi.llama_tokenize(_ctx, text, Encoding.GetEncoding(encoding), tokens, n_ctx, true); + if (n_tokens < 0) + { + throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}"); + } + return tokens.Take(n_tokens).ToList(); + } + + /// + /// Detokenize a list of tokens. + /// + /// The list of tokens to detokenize. + /// The detokenized string. + public string DeTokenize(IEnumerable tokens) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + string output = ""; + foreach (var token in tokens) + { + output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)); + } + return output; + } + + /// + /// Call the model to run inference. + /// + /// + /// + /// + /// + public IEnumerable Call(string text, string encoding = "UTF-8") + { + _is_antiprompt = false; + if (_n_past > 0) + { + _is_interacting = false; + } + if (_is_interacting) + { + if (_verbose) + { + LLamaLogger.Default.Warn("In interacting when calling the model, automatically changed it."); + } + _is_interacting = false; + } + ProcessTextBeforeInfer(text, encoding); + + while ((_n_remain != 0 || _params.interactive) && !_is_interacting) + { + if (_embed.Count > 0) + { + // infinite text generation via context swapping + // if we run out of context: + // - take the n_keep first tokens from the original prompt (via n_past) + // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches + if (_n_past + _embed.Count > _n_ctx) + { + int n_left = _n_past - _params.n_keep; + + _n_past = Math.Max(1, _params.n_keep); + + // insert n_left/2 tokens at the start of embed from last_n_tokens + _embed.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embed.Count).Skip(_n_ctx - n_left / 2 - _embed.Count)); + + // stop saving session if we run out of context + _path_session = ""; + } + + // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) + // REVIEW + if (_n_session_consumed < _session_tokens.Count) + { + int i = 0; + for (; i < _embed.Count; i++) + { + if (_embed[i] != _session_tokens[_n_session_consumed]) + { + _session_tokens = _session_tokens.Take(_n_session_consumed).ToList(); + break; + } + + _n_past++; + _n_session_consumed++; + + if (_n_session_consumed >= _session_tokens.Count) + { + i++; + break; + } + } + + if (i > 0) + { + _embed.RemoveRange(0, i); + } + } + + // evaluate tokens in batches + // embed is typically prepared beforehand to fit within a batch, but not always + for (int i = 0; i < _embed.Count; i += _params.n_batch) + { + int n_eval = _embed.Count - i; + + if (n_eval > _params.n_batch) + { + n_eval = _params.n_batch; + } + + var array = _embed.Skip(i).ToArray(); + if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0) + { + LLamaLogger.Default.Error($"Failed to eval."); + throw new RuntimeError("Failed to eval."); + } + + _n_past += n_eval; + } + + if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session)) + { + _session_tokens.AddRange(_embed); + _n_session_consumed = _session_tokens.Count; + } + } + + _embed.Clear(); + + if (_embed_inp.Count <= _n_consumed && !_is_interacting) + { + var temp = _params.temp; + var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k; + var top_p = _params.top_p; + var tfs_z = _params.tfs_z; + var typical_p = _params.typical_p; + var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n; + var repeat_penalty = _params.repeat_penalty; + var alpha_presence = _params.presence_penalty; + var alpha_frequency = _params.frequency_penalty; + var mirostat = _params.mirostat; + var mirostat_tau = _params.mirostat_tau; + var mirostat_eta = _params.mirostat_eta; + var penalize_nl = _params.penalize_nl; + + // optionally save the session on first sample (for faster prompt loading next time) + if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session) + { + _need_to_save_session = false; + NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count); + } + + llama_token id = 0; + + { + var n_vocab = NativeApi.llama_n_vocab(_ctx); + var logits = Utils.llama_get_logits(_ctx, n_vocab); + + // Apply params.logit_bias map + foreach (var (key, value) in _params.logit_bias) + { + logits[key] += value; + } + + var candidates = new List(); + candidates.Capacity = n_vocab; + for (llama_token token_id = 0; token_id < n_vocab; token_id++) + { + candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f)); + } + + LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false); + + // Apply penalties + float nl_logit = logits[NativeApi.llama_token_nl()]; + var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx); + SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p, + _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(), + (ulong)last_n_repeat, repeat_penalty); + SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p, + _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(), + (ulong)last_n_repeat, alpha_frequency, alpha_presence); + if (!penalize_nl) + { + logits[NativeApi.llama_token_nl()] = nl_logit; + } + + if (temp <= 0) + { + // Greedy sampling + id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p); + } + else + { + if (mirostat == 1) + { + float mirostat_mu = 2.0f * mirostat_tau; + const int mirostat_m = 100; + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, ref mirostat_mu); + } + else if (mirostat == 2) + { + float mirostat_mu = 2.0f * mirostat_tau; + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, ref mirostat_mu); + } + else + { + // Temperature sampling + SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1); + SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1); + SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1); + SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1); + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token(_ctx, candidates_p); + } + } + + _last_n_tokens.RemoveAt(0); + _last_n_tokens.Add(id); + } + + // replace end of text token with newline token when in interactive mode + if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct) + { + id = _llama_token_newline[0]; + if (_params.antiprompt.Count != 0) + { + // tokenize and inject first reverse prompt + var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false, encoding); + _embed_inp.AddRange(first_antiprompt); + } + } + + // add it to the context + _embed.Add(id); + + // echo this to console + _input_echo = true; + + // decrement remaining sampling budget + _n_remain--; + } + else + { + while (_embed_inp.Count > _n_consumed) + { + _embed.Add(_embed_inp[_n_consumed]); + _last_n_tokens.RemoveAt(0); + _last_n_tokens.Add(_embed_inp[_n_consumed]); + _n_consumed++; + if (_embed.Count >= _params.n_batch) + { + break; + } + } + } + + if (_input_echo && !_is_interacting) + { + foreach (var id in _embed) + { + var res = Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); + yield return res; + } + } + + if (_params.interactive && _embed_inp.Count <= _n_consumed) + { + if (_params.antiprompt.Count > 0) + { + string last_output = ""; + foreach (var id in _last_n_tokens) + { + last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); + } + + _is_antiprompt = false; + foreach (var antiprompt in _params.antiprompt) + { + if (last_output.EndsWith(antiprompt)) + { + _is_interacting = true; + _is_antiprompt = true; + break; + } + } + } + + if (_n_past > 0 && _is_interacting) + { + if (_params.instruct) + { + yield return "\n> "; + } + _input_echo = false; + break; + } + + if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos()) + { + if (_params.instruct) + { + _is_interacting = true; + } + else + { + LLamaLogger.Default.Info(" [end of text]"); + } + } + + if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1) + { + _n_remain = _params.n_predict; + _is_interacting = true; + } + } + } + + if (!string.IsNullOrEmpty(_path_session) && _params.prompt_cache_all) + { + LLamaLogger.Default.Info($"saving final output to session file {_path_session}"); + var session_token_array = _session_tokens.ToArray(); + NativeApi.llama_save_session_file(_ctx, _path_session, session_token_array, (ulong)session_token_array.Length); + } + } + + public void Dispose() + { + _ctx.Dispose(); + } + } +} diff --git a/LLama/LLamaParams.cs b/LLama/Old/LLamaParams.cs similarity index 99% rename from LLama/LLamaParams.cs rename to LLama/Old/LLamaParams.cs index e6ee8cc2..7c349c83 100644 --- a/LLama/LLamaParams.cs +++ b/LLama/Old/LLamaParams.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace LLama +namespace LLama.Old { using llama_token = Int32; public struct LLamaParams @@ -66,7 +66,7 @@ namespace LLama string path_session = "", string input_prefix = "", string input_suffix = "", List antiprompt = null, string lora_adapter = "", string lora_base = "", bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false, - bool prompt_cache_all = false, bool embedding = false, bool interactive_first = false, + bool prompt_cache_all = false, bool embedding = false, bool interactive_first = false, bool instruct = false, bool penalize_nl = true, bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false, bool verbose_prompt = false) diff --git a/LLama/LLamaTypes.cs b/LLama/Old/LLamaTypes.cs similarity index 98% rename from LLama/LLamaTypes.cs rename to LLama/Old/LLamaTypes.cs index cee9338f..ae1d4af7 100644 --- a/LLama/LLamaTypes.cs +++ b/LLama/Old/LLamaTypes.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace LLama.Types +namespace LLama.Old { public enum ChatRole { diff --git a/LLama/Old/Utils.cs b/LLama/Old/Utils.cs new file mode 100644 index 00000000..f7203802 --- /dev/null +++ b/LLama/Old/Utils.cs @@ -0,0 +1,98 @@ +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Text; +using LLama.Exceptions; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.IO; + +namespace LLama.Old +{ + using llama_token = Int32; + internal static class Utils + { + public static SafeLLamaContextHandle llama_init_from_gpt_params(ref LLamaParams @params) + { + var lparams = NativeApi.llama_context_default_params(); + + lparams.n_ctx = @params.n_ctx; + lparams.n_gpu_layers = @params.n_gpu_layers; + lparams.seed = @params.seed; + lparams.f16_kv = @params.memory_f16; + lparams.use_mmap = @params.use_mmap; + lparams.use_mlock = @params.use_mlock; + lparams.logits_all = @params.perplexity; + lparams.embedding = @params.embedding; + + if (!File.Exists(@params.model)) + { + throw new FileNotFoundException($"The model file does not exist: {@params.model}"); + } + + var ctx_ptr = NativeApi.llama_init_from_file(@params.model, lparams); + + if (ctx_ptr == IntPtr.Zero) + { + throw new RuntimeError($"Failed to load model {@params.model}."); + } + + SafeLLamaContextHandle ctx = new(ctx_ptr); + + if (!string.IsNullOrEmpty(@params.lora_adapter)) + { + int err = NativeApi.llama_apply_lora_from_file(ctx, @params.lora_adapter, + string.IsNullOrEmpty(@params.lora_base) ? null : @params.lora_base, @params.n_threads); + if (err != 0) + { + throw new RuntimeError("Failed to apply lora adapter."); + } + } + return ctx; + } + + public static List llama_tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, string encodingName) + { + var encoding = Encoding.GetEncoding(encodingName); + var cnt = encoding.GetByteCount(text); + llama_token[] res = new llama_token[cnt + (add_bos ? 1 : 0)]; + int n = NativeApi.llama_tokenize(ctx, text, encoding, res, res.Length, add_bos); + if (n < 0) + { + throw new RuntimeError("Error happened during tokenization. It's possibly caused by wrong encoding. Please try to " + + "specify the encoding."); + } + return res.Take(n).ToList(); + } + + public unsafe static Span llama_get_logits(SafeLLamaContextHandle ctx, int length) + { + var logits = NativeApi.llama_get_logits(ctx); + return new Span(logits, length); + } + + public static unsafe string PtrToStringUTF8(IntPtr ptr) + { +#if NET6_0_OR_GREATER + return Marshal.PtrToStringUTF8(ptr); +#else + byte* tp = (byte*)ptr.ToPointer(); + List bytes = new(); + while (true) + { + byte c = *tp++; + if (c == '\0') + { + break; + } + else + { + bytes.Add(c); + } + } + return Encoding.UTF8.GetString(bytes.ToArray()); +#endif + } + } +} diff --git a/LLama/Utils.cs b/LLama/Utils.cs index 911ad61e..30587b2a 100644 --- a/LLama/Utils.cs +++ b/LLama/Utils.cs @@ -1,50 +1,50 @@ -using LLama.Native; +using LLama.Abstractions.Params; +using LLama.Exceptions; +using LLama.Native; using System; using System.Collections.Generic; -using System.Text; -using LLama.Exceptions; -using System.Diagnostics; +using System.IO; using System.Linq; using System.Runtime.InteropServices; -using System.IO; +using System.Text; namespace LLama { using llama_token = Int32; internal static class Utils { - public static SafeLLamaContextHandle llama_init_from_gpt_params(ref LLamaParams @params) + public static SafeLLamaContextHandle InitLLamaContextFromModelParams(ModelParams @params) { var lparams = NativeApi.llama_context_default_params(); - lparams.n_ctx = @params.n_ctx; - lparams.n_gpu_layers = @params.n_gpu_layers; - lparams.seed = @params.seed; - lparams.f16_kv = @params.memory_f16; - lparams.use_mmap = @params.use_mmap; - lparams.use_mlock = @params.use_mlock; - lparams.logits_all = @params.perplexity; - lparams.embedding = @params.embedding; + lparams.n_ctx = @params.ContextSize; + lparams.n_gpu_layers = @params.GpuLayerCount; + lparams.seed = @params.Seed; + lparams.f16_kv = @params.UseFp16Memory; + lparams.use_mmap = @params.UseMemoryLock; + lparams.use_mlock = @params.UseMemoryLock; + lparams.logits_all = @params.Perplexity; + lparams.embedding = @params.EmbeddingMode; - if (!File.Exists(@params.model)) + if (!File.Exists(@params.ModelPath)) { - throw new FileNotFoundException($"The model file does not exist: {@params.model}"); + throw new FileNotFoundException($"The model file does not exist: {@params.ModelPath}"); } - var ctx_ptr = NativeApi.llama_init_from_file(@params.model, lparams); + var ctx_ptr = NativeApi.llama_init_from_file(@params.ModelPath, lparams); - if(ctx_ptr == IntPtr.Zero ) + if (ctx_ptr == IntPtr.Zero) { - throw new RuntimeError($"Failed to load model {@params.model}."); + throw new RuntimeError($"Failed to load model {@params.ModelPath}."); } SafeLLamaContextHandle ctx = new(ctx_ptr); - if (!string.IsNullOrEmpty(@params.lora_adapter)) + if (!string.IsNullOrEmpty(@params.LoraAdapter)) { - int err = NativeApi.llama_apply_lora_from_file(ctx, @params.lora_adapter, - string.IsNullOrEmpty(@params.lora_base) ? null : @params.lora_base, @params.n_threads); - if(err != 0) + int err = NativeApi.llama_apply_lora_from_file(ctx, @params.LoraAdapter, + string.IsNullOrEmpty(@params.LoraBase) ? null : @params.LoraBase, @params.Threads); + if (err != 0) { throw new RuntimeError("Failed to apply lora adapter."); } @@ -52,37 +52,62 @@ namespace LLama return ctx; } - public static List llama_tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, string encodingName) + public static IEnumerable Tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, Encoding encoding) { - var encoding = Encoding.GetEncoding(encodingName); var cnt = encoding.GetByteCount(text); llama_token[] res = new llama_token[cnt + (add_bos ? 1 : 0)]; int n = NativeApi.llama_tokenize(ctx, text, encoding, res, res.Length, add_bos); - if(n < 0) + if (n < 0) { throw new RuntimeError("Error happened during tokenization. It's possibly caused by wrong encoding. Please try to " + "specify the encoding."); } - return res.Take(n).ToList(); + return res.Take(n); } - public unsafe static Span llama_get_logits(SafeLLamaContextHandle ctx, int length) + public unsafe static Span GetLogits(SafeLLamaContextHandle ctx, int length) { var logits = NativeApi.llama_get_logits(ctx); return new Span(logits, length); } - public static unsafe string PtrToStringUTF8(IntPtr ptr) + public static unsafe int Eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int startIndex, int n_tokens, int n_past, int n_threads) + { + int result; + fixed(llama_token* p = tokens) + { + result = NativeApi.llama_eval_with_pointer(ctx, p + startIndex, n_tokens, n_past, n_threads); + } + return result; + } + + public static string TokenToString(llama_token token, SafeLLamaContextHandle ctx, Encoding encoding) + { + return PtrToString(NativeApi.llama_token_to_str(ctx, token), encoding); + } + + public static unsafe string PtrToString(IntPtr ptr, Encoding encoding) { #if NET6_0_OR_GREATER - return Marshal.PtrToStringUTF8(ptr); + if(encoding == Encoding.UTF8) + { + return Marshal.PtrToStringUTF8(ptr); + } + else if(encoding == Encoding.Unicode) + { + return Marshal.PtrToStringUni(ptr); + } + else + { + return Marshal.PtrToStringAuto(ptr); + } #else byte* tp = (byte*)ptr.ToPointer(); List bytes = new(); while (true) { byte c = *tp++; - if(c == '\0') + if (c == '\0') { break; } @@ -91,7 +116,7 @@ namespace LLama bytes.Add(c); } } - return Encoding.UTF8.GetString(bytes.ToArray()); + return encoding.GetString(bytes.ToArray()); #endif } } diff --git a/LLama/libllama.dll b/LLama/libllama.dll deleted file mode 100644 index a5d0ba40..00000000 Binary files a/LLama/libllama.dll and /dev/null differ diff --git a/LLama/libllama.so b/LLama/libllama.so deleted file mode 100644 index f1f0037e..00000000 Binary files a/LLama/libllama.so and /dev/null differ