From 9c0f3aedbabea700ae458dcdc3319d309928de8e Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Tue, 16 May 2023 02:55:25 +0800 Subject: [PATCH] refactor: change some file names. --- LLama/GptModel.cs | 588 --------------------- LLama/LLamaModel.cs | 1153 ++++++++++++++++------------------------- LLama/LLamaModelV1.cs | 837 ++++++++++++++++++++++++++++++ 3 files changed, 1289 insertions(+), 1289 deletions(-) delete mode 100644 LLama/GptModel.cs create mode 100644 LLama/LLamaModelV1.cs diff --git a/LLama/GptModel.cs b/LLama/GptModel.cs deleted file mode 100644 index f29725b4..00000000 --- a/LLama/GptModel.cs +++ /dev/null @@ -1,588 +0,0 @@ -using LLama.Native; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using LLama.Exceptions; -using System.Linq; -using System.Text.RegularExpressions; -using System.Runtime.InteropServices; -using System.Diagnostics; - -namespace LLama -{ - using llama_token = Int32; - public class LLamaModel: IChatModel, IDisposable - { - LLamaParams _params; - SafeLLamaContextHandle _ctx; - string _path_session; - List _session_tokens; - List _embed_inp; - int _n_ctx; - List _inp_pfx; - List _inp_sfx; - List _llama_token_newline; - List _last_n_tokens; - bool _is_interacting; - bool _is_antiprompt; - bool _input_echo; - - // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session - // if we loaded a session with at least 75% similarity. It's currently just used to speed up the - // initial prompt so it doesn't need to be an exact match. - bool _need_to_save_session; - int _n_past; - int _n_remain; - int _n_consumed; - int _n_session_consumed; - List _embed; - - // params related to chat API only - bool _first_time_chat = true; - - public string Name { get; set; } - public SafeLLamaContextHandle NativeHandle => _ctx; - - public LLamaModel(string model_path, string model_name, bool echo_input = false, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, - int n_parts = -1, int n_ctx = 512, int n_batch = 512, int n_keep = 0, - Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, - float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, - int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f, - int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "", - string path_session = "", string input_prefix = "", string input_suffix = "", - List antiprompt = null, string lora_adapter = "", string lora_base = "", - bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false, - bool embedding = false, bool interactive_first = false, bool instruct = false, bool penalize_nl = true, - bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false, - bool verbose_prompt = false) : this(new LLamaParams(seed, n_threads, n_predict, n_parts, n_ctx, n_batch, - n_keep, logit_bias, top_k, top_p, tfs_z, typical_p, temp, repeat_penalty, repeat_last_n, frequency_penalty, - presence_penalty, mirostat, mirostat_tau, mirostat_eta, model_path, prompt, path_session, input_prefix, - input_suffix, antiprompt, lora_adapter, lora_base, memory_f16, random_prompt, use_color, interactive, embedding, - interactive_first, instruct, penalize_nl, perplexity, use_mmap, use_mlock, mem_test, verbose_prompt), model_name, echo_input, verbose) - { - - } - - public unsafe LLamaModel(LLamaParams @params, string name = "", bool echo_input = false, bool verbose = false) - { - Name = name; - _params = @params; - _ctx = Utils.llama_init_from_gpt_params(ref _params); - - // Add a space in front of the first character to match OG llama tokenizer behavior - _params.prompt = _params.prompt.Insert(0, " "); - _session_tokens = new List(); - - _path_session = @params.path_session; - if (!string.IsNullOrEmpty(_path_session)) - { - if (verbose) - { - Logger.Default.Info($"Attempting to load saved session from '{_path_session}'"); - } - - if (!File.Exists(_path_session)) - { - Logger.Default.Warn("Session file does not exist, will create."); - } - - llama_token[] session_tokens = new llama_token[@params.n_ctx]; - ulong n_token_count_out = 0; - if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out)) - { - throw new RuntimeError($"Failed to load session file {_path_session}"); - } - _session_tokens = session_tokens.Take((int)n_token_count_out).ToList(); - if (verbose) - { - Logger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens"); - } - } - - _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true); - _n_ctx = NativeApi.llama_n_ctx(_ctx); - - if (_embed_inp.Count > _n_ctx - 4) - { - throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); - } - - ulong n_matching_session_tokens = 0; - if (_session_tokens.Count > 0) - { - foreach (var id in _session_tokens) - { - if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens]) - { - break; - } - n_matching_session_tokens++; - } - if (n_matching_session_tokens >= (ulong)_embed_inp.Count && verbose) - { - Logger.Default.Info("Session file has exact match for prompt!"); - } - else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2)) - { - Logger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " + - $"/ {_embed_inp.Count} tokens); will mostly be reevaluated."); - } - else if(verbose) - { - Logger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " + - $"tokens of prompt."); - } - } - - // number of tokens to keep when resetting context - if (_params.n_keep < 0 || _params.n_keep > (int)_embed_inp.Count || _params.instruct) - { - _params.n_keep = _embed_inp.Count; - } - - // prefix & suffix for instruct mode - _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true); - _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false); - - // in instruct mode, we inject a prefix and a suffix to each input by the user - if (_params.instruct) - { - _params.interactive_first = true; - _params.antiprompt.Add("### Instruction:\n\n"); - } - - // enable interactive mode if reverse prompt or interactive start is specified - if (_params.antiprompt.Count != 0 || _params.interactive_first) - { - _params.interactive = true; - } - - // determine newline token - _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false); - - if (_params.verbose_prompt) - { - Logger.Default.Info("\n"); - Logger.Default.Info($"prompt: '{_params.prompt}'"); - Logger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}"); - for (int i = 0; i < _embed_inp.Count; i++) - { - Logger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'"); - } - if (_params.n_keep > 0) - { - Logger.Default.Info($"static prompt based on n_keep: '"); - for (int i = 0; i < _params.n_keep; i++) - { - Logger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}"); - } - Logger.Default.Info("\n"); - } - Logger.Default.Info("\n"); - } - - if (_params.interactive && verbose) - { - Logger.Default.Info("interactive mode on."); - } - if (verbose) - { - Logger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " + - $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + - $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + - $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + - $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); - Logger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " + - $"n_keep = {_params.n_keep}"); - Logger.Default.Info("\n"); - } - - _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList(); - - if (_params.interactive) - { - if (verbose) - { - Logger.Default.Info("== Running in interactive mode. =="); - } - _is_interacting = _params.interactive_first; - } - - _is_antiprompt = false; - _input_echo = echo_input; - _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4); - _n_past = 0; - _n_remain = _params.n_predict; - _n_consumed = 0; - _n_session_consumed = 0; - _embed = new List(); - } - - public LLamaModel WithPrompt(string prompt) - { - _params.prompt = prompt; - if (!_params.prompt.EndsWith(" ")) - { - _params.prompt = _params.prompt.Insert(0, " "); - } - _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true); - if (_embed_inp.Count > _n_ctx - 4) - { - throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); - } - return this; - } - - public LLamaModel WithPromptFile(string promptFileName) - { - return WithPrompt(File.ReadAllText(promptFileName)); - } - - private string ProcessTextBeforeInfer(string text) - { - if (!string.IsNullOrEmpty(_params.input_prefix)) - { - text = _params.input_prefix + text; - } - if (!text.EndsWith("\n")) - { - text += "\n"; - } - if (text.Length > 1) - { - // append input suffix if any - if (!string.IsNullOrEmpty(_params.input_suffix)) - { - text += _params.input_suffix; - Console.Write(_params.input_suffix); - } - - // instruct mode: insert instruction prefix - if (_params.instruct && !_is_antiprompt) - { - _n_consumed = _embed_inp.Count; - _embed_inp.AddRange(_inp_pfx); - } - - var line_inp = Utils.llama_tokenize(_ctx, text, false); - _embed_inp.AddRange(line_inp); - - // instruct mode: insert response suffix - if (_params.instruct) - { - _embed_inp.AddRange(_inp_sfx); - } - - _n_remain -= line_inp.Count; - } - return text; - } - - public void InitChatPrompt(string prompt) - { - WithPrompt(prompt); - } - - public void InitChatAntiprompt(string[] antiprompt) - { - _params.antiprompt = antiprompt.ToList(); - } - - public IEnumerable Chat(string text, string? prompt = null) - { - _params.interactive = true; - _input_echo = false; - if (!string.IsNullOrEmpty(prompt)) - { - WithPrompt(prompt); - } - return Call(text); - } - - public IEnumerable Call(string text) - { - _is_interacting = _is_antiprompt = false; - ProcessTextBeforeInfer(text); - - while ((_n_remain != 0 || _params.interactive) && !_is_interacting) - { - if (_embed.Count > 0) - { - // infinite text generation via context swapping - // if we run out of context: - // - take the n_keep first tokens from the original prompt (via n_past) - // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches - if (_n_past + _embed.Count > _n_ctx) - { - int n_left = _n_past - _params.n_keep; - - _n_past = _params.n_keep; - - // insert n_left/2 tokens at the start of embed from last_n_tokens - _embed.InsertRange(0, _last_n_tokens.GetRange(_n_ctx - n_left / 2 - _embed.Count, _embed.Count)); - - // stop saving session if we run out of context - _path_session = ""; - - // Console.WriteLine("\n---\n"); - // Console.Write("resetting: '"); - // for (int i = 0; i < embed.Count; i++) { - // Console.Write(llama_token_to_str(ctx, embed[i])); - // } - // Console.WriteLine("'\n"); - // Console.WriteLine("\n---\n"); - } - - // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) - // REVIEW - if (_n_session_consumed < _session_tokens.Count) - { - int i = 0; - for (; i < _embed.Count; i++) - { - if (!_embed[i].Equals(_session_tokens[_n_session_consumed])) - { - _session_tokens.RemoveRange(_n_session_consumed, _session_tokens.Count - _n_session_consumed); - break; - } - - _n_past++; - _n_session_consumed++; - - if (_n_session_consumed >= _session_tokens.Count) - { - i++; - break; - } - } - - if (i > 0) - { - _embed.RemoveRange(0, i); - } - } - - // evaluate tokens in batches - // embed is typically prepared beforehand to fit within a batch, but not always - for (int i = 0; i < _embed.Count; i += _params.n_batch) - { - int n_eval = _embed.Count - i; - - if (n_eval > _params.n_batch) - { - n_eval = _params.n_batch; - } - - var array = _embed.GetRange(i, n_eval).ToArray(); - if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0) - { - Logger.Default.Error($"Failed to eval"); - throw new RuntimeError("Failed to eval"); - } - - _n_past += n_eval; - } - - if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session)) - { - _session_tokens.AddRange(_embed); - _n_session_consumed = _session_tokens.Count; - } - } - - _embed.Clear(); - - if (_embed_inp.Count <= _n_consumed && !_is_interacting) - { - var temp = _params.temp; - var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k; - var top_p = _params.top_p; - var tfs_z = _params.tfs_z; - var typical_p = _params.typical_p; - var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n; - var repeat_penalty = _params.repeat_penalty; - var alpha_presence = _params.presence_penalty; - var alpha_frequency = _params.frequency_penalty; - var mirostat = _params.mirostat; - var mirostat_tau = _params.mirostat_tau; - var mirostat_eta = _params.mirostat_eta; - var penalize_nl = _params.penalize_nl; - - // optionally save the session on first sample (for faster prompt loading next time) - if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session) - { - _need_to_save_session = false; - NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count); - } - - llama_token id = 0; - - { - var n_vocab = NativeApi.llama_n_vocab(_ctx); - var logits = Utils.llama_get_logits(_ctx, n_vocab); - - // Apply params.logit_bias map - foreach (KeyValuePair it in _params.logit_bias) - { - logits[it.Key] += it.Value; - } - - var candidates = new List(); - candidates.Capacity = n_vocab; - for (llama_token token_id = 0; token_id < n_vocab; token_id++) - { - candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f)); - } - - LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false); - - // Apply penalties - float nl_logit = logits[NativeApi.llama_token_nl()]; - var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx); - SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p, - _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(), - (ulong)last_n_repeat, repeat_penalty); - SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p, - _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(), - (ulong)last_n_repeat, alpha_frequency, alpha_presence); - if (!penalize_nl) - { - logits[NativeApi.llama_token_nl()] = nl_logit; - } - - if (temp <= 0) - { - // Greedy sampling - id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p); - } - else - { - if (mirostat == 1) - { - float mirostat_mu = 2.0f * mirostat_tau; - const int mirostat_m = 100; - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, mirostat_mu); - } - else if (mirostat == 2) - { - float mirostat_mu = 2.0f * mirostat_tau; - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_mu); - } - else - { - // Temperature sampling - SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1); - SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1); - SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1); - SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1); - SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); - id = SamplingApi.llama_sample_token(_ctx, candidates_p); - } - } - - _last_n_tokens.RemoveAt(0); - _last_n_tokens.Add(id); - } - - // replace end of text token with newline token when in interactive mode - if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct) - { - id = _llama_token_newline[0]; - if (_params.antiprompt.Count != 0) - { - // tokenize and inject first reverse prompt - var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false); - _embed_inp.AddRange(first_antiprompt); - } - } - - // add it to the context - _embed.Add(id); - - // echo this to console - _input_echo = true; - - // decrement remaining sampling budget - _n_remain--; - } - else - { - // Assuming that the necessary variables have been defined and initialized, - // the C# equivalent code could be: - - while (_embed_inp.Count > _n_consumed) - { - _embed.Add(_embed_inp[_n_consumed]); - _last_n_tokens.RemoveAt(0); - _last_n_tokens.Add(_embed_inp[_n_consumed]); - _n_consumed++; - if (_embed.Count >= _params.n_batch) - { - break; - } - } - } - - if (_input_echo) - { - foreach (var id in _embed) - { - yield return Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); - } - } - - if (_params.interactive && _embed_inp.Count <= _n_consumed) - { - if (_params.antiprompt.Count > 0) - { - string last_output = ""; - foreach (var id in _last_n_tokens) - { - last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); - } - - _is_antiprompt = false; - foreach (var antiprompt in _params.antiprompt) - { - if (last_output.EndsWith(antiprompt)) - { - _is_interacting = true; - _is_antiprompt = true; - break; - } - } - } - - if(_n_past > 0 && _is_interacting) - { - _input_echo = false; - break; - } - - if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos()) - { - if (_params.instruct) { - _is_interacting = true; - } else - { - Logger.Default.Info(" [end of text]"); - } - } - - if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1) { - _n_remain = _params.n_predict; - _is_interacting = true; - } - } - } - } - - public void Dispose() - { - _ctx.Dispose(); - } - } -} diff --git a/LLama/LLamaModel.cs b/LLama/LLamaModel.cs index c984a63d..f29725b4 100644 --- a/LLama/LLamaModel.cs +++ b/LLama/LLamaModel.cs @@ -1,832 +1,583 @@ -using LLama.Exceptions; -using LLama.Native; +using LLama.Native; using System; using System.Collections.Generic; -using System.Configuration; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; using System.Text; -using LLama.Types; -using System.Runtime.InteropServices; +using LLama.Exceptions; +using System.Linq; using System.Text.RegularExpressions; -using System.Collections; +using System.Runtime.InteropServices; +using System.Diagnostics; namespace LLama { using llama_token = Int32; - /// - /// High-level Wrapper of a llama.cpp model for inference. Note that it's more recommended to use `LLamaModel`. - /// This class may be removed in the future. However, if all you want is to get the embeddings, then using `LLamaModelV1` - /// is ok now. - /// - [Obsolete] - public class LLamaModelV1: IDisposable + public class LLamaModel: IChatModel, IDisposable { - private string _model_path; - LLamaContextParams _params; - private int _n_threads; - private int _n_batch; - private int _last_n_tokens_size; - private string? _lora_base; - private string? _lora_path; - private bool _verbose; - - private Queue _eval_tokens; - private Queue _eval_logits; - private LLamaCache? _cache; - private SafeLLamaContextHandle _ctx; - - private static readonly (int, int)[] _numAndPatterns = new (int, int)[] { (2, 192), (3, 224), (4, 240) }; - - /// - /// Load a llama.cpp model from the path. - /// - /// Note that the API is still unstable. The order of them is likely to - /// be changed in the future. It's recommened to specify the parameter name when - /// building your app. We use the cpp style parameter names here because it introduces - /// convenience for searching the docs. - /// Path to the model. - /// Maximum context size. - /// Number of parts to split the model into. If -1, the number of parts is automatically determined. - /// Random seed. 0 for random. - /// Use half-precision for key/value cache. - /// Return logits for all tokens, not just the last token. - /// Only load the vocabulary no weights. - /// Use mmap if possible. - /// Force the system to keep the model in RAM. - /// Embedding mode only. - /// Number of threads to use. If is not specified, the number of threads is automatically determined. - /// Maximum number of prompt tokens to batch together when calling llama_eval. - /// Maximum number of tokens to keep in the last_n_tokens deque. - /// Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. - /// Path to a LoRA file to apply to the model. - /// Print verbose output to stderr. - public LLamaModelV1(string model_path, int n_ctx = 512, int n_parts = -1, int seed = 1337, - bool f16_kv = true, bool logits_all = false, bool vocab_only = false, bool use_mmap = true, - bool use_mlock = false, bool embedding = false, int n_threads = -1, int n_batch = 512, - int last_n_tokens_size = 64, string? lora_base = null, string? lora_path = null, bool verbose = true) + LLamaParams _params; + SafeLLamaContextHandle _ctx; + string _path_session; + List _session_tokens; + List _embed_inp; + int _n_ctx; + List _inp_pfx; + List _inp_sfx; + List _llama_token_newline; + List _last_n_tokens; + bool _is_interacting; + bool _is_antiprompt; + bool _input_echo; + + // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session + // if we loaded a session with at least 75% similarity. It's currently just used to speed up the + // initial prompt so it doesn't need to be an exact match. + bool _need_to_save_session; + int _n_past; + int _n_remain; + int _n_consumed; + int _n_session_consumed; + List _embed; + + // params related to chat API only + bool _first_time_chat = true; + + public string Name { get; set; } + public SafeLLamaContextHandle NativeHandle => _ctx; + + public LLamaModel(string model_path, string model_name, bool echo_input = false, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, + int n_parts = -1, int n_ctx = 512, int n_batch = 512, int n_keep = 0, + Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, + float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, + int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f, + int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "", + string path_session = "", string input_prefix = "", string input_suffix = "", + List antiprompt = null, string lora_adapter = "", string lora_base = "", + bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false, + bool embedding = false, bool interactive_first = false, bool instruct = false, bool penalize_nl = true, + bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false, + bool verbose_prompt = false) : this(new LLamaParams(seed, n_threads, n_predict, n_parts, n_ctx, n_batch, + n_keep, logit_bias, top_k, top_p, tfs_z, typical_p, temp, repeat_penalty, repeat_last_n, frequency_penalty, + presence_penalty, mirostat, mirostat_tau, mirostat_eta, model_path, prompt, path_session, input_prefix, + input_suffix, antiprompt, lora_adapter, lora_base, memory_f16, random_prompt, use_color, interactive, embedding, + interactive_first, instruct, penalize_nl, perplexity, use_mmap, use_mlock, mem_test, verbose_prompt), model_name, echo_input, verbose) { - _verbose = verbose; - _model_path = model_path; - - _params = NativeApi.llama_context_default_params(); - _params.n_ctx = n_ctx; - _params.n_parts = n_parts; - _params.seed = seed; - _params.f16_kv = f16_kv; - _params.logits_all = logits_all; - _params.vocab_only = vocab_only; - _params.use_mmap = lora_path is null ? use_mmap : false; - _params.use_mlock = use_mlock; - _params.embedding = embedding; - - _last_n_tokens_size = last_n_tokens_size; - _n_batch = Math.Min(n_ctx, n_batch); - - _eval_tokens = new Queue(capacity: n_ctx); - _eval_logits = new Queue(logits_all ? n_ctx : 1); - - _cache = null; - - _n_threads = n_threads; - if(_n_threads == -1) - { - _n_threads = Math.Max(Environment.ProcessorCount / 2, 1); - } - _lora_base = lora_base; - _lora_path = lora_path; + } + + public unsafe LLamaModel(LLamaParams @params, string name = "", bool echo_input = false, bool verbose = false) + { + Name = name; + _params = @params; + _ctx = Utils.llama_init_from_gpt_params(ref _params); + + // Add a space in front of the first character to match OG llama tokenizer behavior + _params.prompt = _params.prompt.Insert(0, " "); + _session_tokens = new List(); - if(!File.Exists(model_path) && !Directory.Exists(model_path)) + _path_session = @params.path_session; + if (!string.IsNullOrEmpty(_path_session)) { - throw new FileNotFoundException($"Model path does not exist: {model_path}"); + if (verbose) + { + Logger.Default.Info($"Attempting to load saved session from '{_path_session}'"); + } + + if (!File.Exists(_path_session)) + { + Logger.Default.Warn("Session file does not exist, will create."); + } + + llama_token[] session_tokens = new llama_token[@params.n_ctx]; + ulong n_token_count_out = 0; + if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out)) + { + throw new RuntimeError($"Failed to load session file {_path_session}"); + } + _session_tokens = session_tokens.Take((int)n_token_count_out).ToList(); + if (verbose) + { + Logger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens"); + } } - // Move from heap to stack to prevent the moving. - _ctx = new SafeLLamaContextHandle(NativeApi.llama_init_from_file(Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(model_path)), _params)); + _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true); + _n_ctx = NativeApi.llama_n_ctx(_ctx); - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + if (_embed_inp.Count > _n_ctx - 4) + { + throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); + } - if(_lora_path is not null) + ulong n_matching_session_tokens = 0; + if (_session_tokens.Count > 0) { - if(NativeApi.llama_apply_lora_from_file(_ctx, lora_path, lora_base, _n_threads) != 0) + foreach (var id in _session_tokens) + { + if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens]) + { + break; + } + n_matching_session_tokens++; + } + if (n_matching_session_tokens >= (ulong)_embed_inp.Count && verbose) + { + Logger.Default.Info("Session file has exact match for prompt!"); + } + else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2)) + { + Logger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " + + $"/ {_embed_inp.Count} tokens); will mostly be reevaluated."); + } + else if(verbose) { - throw new RuntimeError($"Failed to apply LoRA from lora path: {_lora_path} to base path: {_lora_base}"); + Logger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " + + $"tokens of prompt."); } } - if (_verbose) + // number of tokens to keep when resetting context + if (_params.n_keep < 0 || _params.n_keep > (int)_embed_inp.Count || _params.instruct) { - Logger.Default.Info(Utils.PtrToStringUTF8(NativeApi.llama_print_system_info())); + _params.n_keep = _embed_inp.Count; } - } - public LLamaModelV1(LLamaModelV1 other) - { - _ctx = other._ctx; - _model_path = other._model_path; - _params = other._params; - _last_n_tokens_size = other._last_n_tokens_size; - _n_threads = other._n_threads; - _n_batch = other._n_batch; - _verbose = other._verbose; - _lora_base = other._lora_base; - _lora_path = other._lora_path; - _eval_logits = new Queue(other._eval_logits); - _eval_tokens = new Queue(other._eval_tokens); - } + // prefix & suffix for instruct mode + _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true); + _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false); - /// - /// Tokenize a string. - /// - /// The utf-8 encoded string to tokenize. - /// A list of tokens. - /// If the tokenization failed. - public List Tokenize(string text) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - var n_ctx = NativeApi.llama_n_ctx(_ctx); - var tokens = new llama_token[n_ctx]; - var n_tokens = NativeApi.llama_tokenize(_ctx, text, tokens, n_ctx, true); - if(n_tokens < 0) + // in instruct mode, we inject a prefix and a suffix to each input by the user + if (_params.instruct) { - throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}"); + _params.interactive_first = true; + _params.antiprompt.Add("### Instruction:\n\n"); } - return tokens.Take(n_tokens).ToList(); - } - /// - /// Detokenize a list of tokens. - /// - /// The list of tokens to detokenize. - /// The detokenized string. - public string DeTokenize(IEnumerable tokens) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - string output = ""; - foreach(var token in tokens) + // enable interactive mode if reverse prompt or interactive start is specified + if (_params.antiprompt.Count != 0 || _params.interactive_first) { - output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)); + _params.interactive = true; } - return output; - } - public string DeTokenize(llama_token token) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - return Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)) ?? ""; - } - - /// - /// Set the cache. - /// - /// The cache to set. - public void SetCache(LLamaCache? cache) - { - _cache = cache; - } - - /// - /// Reset the model state. - /// - public void Reset() - { - _eval_tokens.Clear(); - _eval_logits.Clear(); - } + // determine newline token + _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false); - /// - /// Evaluate a list of tokens. - /// - /// The list of tokens to evaluate. - /// - public unsafe void Eval(List tokens) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - var n_ctx = NativeApi.llama_n_ctx(_ctx); - for(int i = 0; i < tokens.Count; i += _n_batch) + if (_params.verbose_prompt) { - var batch = tokens.Take(Math.Min(tokens.Count, i + _n_batch)).Skip(i); - llama_token n_past = Math.Min(n_ctx - batch.Count(), _eval_tokens.Count); - llama_token n_tokens = batch.Count(); - llama_token return_code = NativeApi.llama_eval( - ctx: _ctx, - tokens: batch.ToArray(), - n_tokens: n_tokens, - n_past: n_past, - n_threads: _n_threads - ); - if(return_code != 0) - { - throw new RuntimeError($"llama_eval returned {return_code}"); - } - foreach(var b in batch) + Logger.Default.Info("\n"); + Logger.Default.Info($"prompt: '{_params.prompt}'"); + Logger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}"); + for (int i = 0; i < _embed_inp.Count; i++) { - _eval_tokens.Enqueue(b); + Logger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'"); } - int rows = _params.logits_all ? n_tokens : 1; - llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); - var cols = n_vocab; - var logits_view = NativeApi.llama_get_logits(_ctx); - for(int j = 0; j < rows; j++) + if (_params.n_keep > 0) { - float[] logit = new float[cols]; - for(int k = 0; k < cols; k++) + Logger.Default.Info($"static prompt based on n_keep: '"); + for (int i = 0; i < _params.n_keep; i++) { - logit[k] = logits_view[j * cols + k]; + Logger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}"); } - _eval_logits.Enqueue(logit); + Logger.Default.Info("\n"); } + Logger.Default.Info("\n"); } - } - private llama_token SampleInternal(llama_token[] last_n_tokens_data, int last_n_tokens_size, int top_k, - float top_p, float temp, float repeat_penalty, float frequency_penalty, float presence_penalty) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - Debug.Assert(_eval_logits.Count > 0); - llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); - var logits = _eval_logits.Last(); - LLamaTokenData[] data = new LLamaTokenData[n_vocab]; - for(int i = 0; i < n_vocab; i++) - { - data[i] = new LLamaTokenData(i, logits[i], .0f); - } - ulong size = (ulong)n_vocab; - bool sorted = false; - LLamaTokenDataArray candidates = new(data, size, sorted); - SamplingApi.llama_sample_repetition_penalty(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, - repeat_penalty); - //SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, - // frequency_penalty, presence_penalty); - if(temp == .0f) + if (_params.interactive && verbose) { - return SamplingApi.llama_sample_token_greedy(_ctx, candidates); + Logger.Default.Info("interactive mode on."); } - else + if (verbose) { - SamplingApi.llama_sample_top_k(_ctx, candidates, top_k, 1); - SamplingApi.llama_sample_tail_free(_ctx, candidates, 1.0f, 1); - SamplingApi.llama_sample_typical(_ctx, candidates, 1.0f, 1); - SamplingApi.llama_sample_top_p(_ctx, candidates, top_p, 1); - SamplingApi.llama_sample_temperature(_ctx, candidates, temp); - return SamplingApi.llama_sample_token(_ctx, candidates); + Logger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " + + $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + + $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + + $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + + $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); + Logger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " + + $"n_keep = {_params.n_keep}"); + Logger.Default.Info("\n"); } - } - /// - /// Sample a token from the model. - /// - /// The top-k sampling parameter. - /// The top-p sampling parameter. - /// The temperature parameter. - /// The repeat penalty parameter. - /// - /// - /// The sampled token. - public llama_token Sample(int top_k, float top_p, float temp, float repeat_penalty, float frequency_penalty = .0f, - float presence_penalty = .0f) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - var last_n_tokens_data = Enumerable.Repeat(0, Math.Max(0, _last_n_tokens_size - _eval_tokens.Count)); - last_n_tokens_data = last_n_tokens_data.Concat(_eval_tokens.ToList() - .Skip(Math.Max(0, _eval_tokens.Count - _last_n_tokens_size))); - llama_token[] tokens_data = new llama_token[_last_n_tokens_size]; - int i = 0; - foreach(var data in last_n_tokens_data) + _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList(); + + if (_params.interactive) { - if(i < _last_n_tokens_size) - { - tokens_data[i++] = data; - } - else + if (verbose) { - break; + Logger.Default.Info("== Running in interactive mode. =="); } + _is_interacting = _params.interactive_first; } - return SampleInternal(tokens_data, _last_n_tokens_size, top_k, top_p, temp, repeat_penalty, frequency_penalty, presence_penalty); + + _is_antiprompt = false; + _input_echo = echo_input; + _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4); + _n_past = 0; + _n_remain = _params.n_predict; + _n_consumed = 0; + _n_session_consumed = 0; + _embed = new List(); } - /// - /// Create a generator of tokens from a prompt. - /// - /// - /// Examples: - /// var llama = new LlamaModel("models/ggml-7b.bin") - /// var tokens = llama.Tokenize(b"Hello, world!") - /// foreach(var token in llama.Generate(tokens, top_k:40, top_p:0.95, temp:1.0, repeat_penalty:1.1)){ - /// Console.WriteLine(llama.DeTokenize(new []{token})); - /// } - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - public IEnumerable Generate(IEnumerable tokens, int top_k, float top_p, float temp, - float repeat_penalty, float frequency_penalty = .0f, float presence_penalty = .0f, bool reset = true) + public LLamaModel WithPrompt(string prompt) { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - if(reset && _eval_tokens.Count > 0) + _params.prompt = prompt; + if (!_params.prompt.EndsWith(" ")) { - int longest_prefix = 0; - foreach(var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count() - 1), (x, y) => (x, y))) - { - if(a == b) - { - longest_prefix += 1; - } - else - { - break; - } - } - if(longest_prefix > 0) - { - if (_verbose) - { - Logger.Default.Info("Llama.generate: prefix-match hit"); - } - reset = false; - tokens = tokens.Skip(longest_prefix); - for(int i = 0; i < _eval_tokens.Count - longest_prefix; i++) - { - _eval_tokens.Dequeue(); - if(_eval_logits.Count > 0) - { - _eval_logits.Dequeue(); - } - } - } + _params.prompt = _params.prompt.Insert(0, " "); } - - if (reset) + _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true); + if (_embed_inp.Count > _n_ctx - 4) { - Reset(); + throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); } + return this; + } - while (true) - { - Eval(tokens.ToList()); - var token = Sample(top_k, top_p, temp, frequency_penalty, presence_penalty, repeat_penalty); - yield return token; - // TODO(Rinne): verify if the implementation is correct. - } + public LLamaModel WithPromptFile(string promptFileName) + { + return WithPrompt(File.ReadAllText(promptFileName)); } - /// - /// Embed a string. - /// - /// The utf-8 encoded string to embed. - /// An embedding object. - /// - public unsafe Embedding CreateEmbedding(string input) + private string ProcessTextBeforeInfer(string text) { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - if (!_params.embedding) + if (!string.IsNullOrEmpty(_params.input_prefix)) { - throw new RuntimeError("Llama model must be created with embedding=True to call this method"); + text = _params.input_prefix + text; } - - if (_verbose) + if (!text.EndsWith("\n")) { - NativeApi.llama_reset_timings(_ctx); + text += "\n"; } - - var tokens = Tokenize(input); - Reset(); - Eval(tokens); - int n_tokens = tokens.Count; - var embeddingPtr = NativeApi.llama_get_embeddings(_ctx); - int cnt = NativeApi.llama_n_embd(_ctx); - float[] embedding = new float[cnt]; - for(int i = 0; i < cnt; i++) + if (text.Length > 1) { - embedding[i] = embeddingPtr[i]; - } + // append input suffix if any + if (!string.IsNullOrEmpty(_params.input_suffix)) + { + text += _params.input_suffix; + Console.Write(_params.input_suffix); + } - if (_verbose) - { - NativeApi.llama_print_timings(_ctx); - } + // instruct mode: insert instruction prefix + if (_params.instruct && !_is_antiprompt) + { + _n_consumed = _embed_inp.Count; + _embed_inp.AddRange(_inp_pfx); + } + + var line_inp = Utils.llama_tokenize(_ctx, text, false); + _embed_inp.AddRange(line_inp); - return new Embedding("list", _model_path, new[] { new EmbeddingData(0, "embedding", embedding) }, - new EmbeddingUsage(n_tokens, n_tokens)); + // instruct mode: insert response suffix + if (_params.instruct) + { + _embed_inp.AddRange(_inp_sfx); + } + + _n_remain -= line_inp.Count; + } + return text; } - public float[] Embed(string input) + public void InitChatPrompt(string prompt) { - return CreateEmbedding(input).Data[0].Embedding; + WithPrompt(prompt); } - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// IEnumerable of Completion and CompletionChunk - /// - private IEnumerable CreateCompletionInternal(string prompt, string?suffix = null, int max_tokens = 16, float temperature = 0.8f, - float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, - float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + public void InitChatAntiprompt(string[] antiprompt) { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - string completionId = $"cmpl-{Guid.NewGuid()}"; - var created = DateTime.Now.Millisecond; - List completionTokens = new List(); - - var promptTokens = Tokenize($" {prompt}"); - string text = ""; - int returnedCharacters = 0; - if(stop is null) - { - stop = new string[0]; - } + _params.antiprompt = antiprompt.ToList(); + } - if (_verbose) + public IEnumerable Chat(string text, string? prompt = null) + { + _params.interactive = true; + _input_echo = false; + if (!string.IsNullOrEmpty(prompt)) { - NativeApi.llama_reset_timings(_ctx); + WithPrompt(prompt); } + return Call(text); + } - if(promptTokens.Count + max_tokens > NativeApi.llama_n_ctx(_ctx)) - { - throw new ArgumentException($"Requested tokens exceed context window of {NativeApi.llama_n_ctx(_ctx)}"); - } + public IEnumerable Call(string text) + { + _is_interacting = _is_antiprompt = false; + ProcessTextBeforeInfer(text); - if(logprobs != -1 && !_params.logits_all) + while ((_n_remain != 0 || _params.interactive) && !_is_interacting) { - throw new ArgumentException("logprobs is not supported for models created with logits_all=False"); - } - - if(_cache is not null) - { - try + if (_embed.Count > 0) { - // TODO(Rinne): revise it since it will compare reference instead of elements. - var cacheItem = _cache[promptTokens.ToArray()]; - var cachePrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); - var evalPrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); - if(cachePrefixLen > evalPrefixLen) + // infinite text generation via context swapping + // if we run out of context: + // - take the n_keep first tokens from the original prompt (via n_past) + // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches + if (_n_past + _embed.Count > _n_ctx) { - LoadState(cacheItem); - if (_verbose) - { - Logger.Default.Info("Llama._create_completion: cache hit"); - } + int n_left = _n_past - _params.n_keep; + + _n_past = _params.n_keep; + + // insert n_left/2 tokens at the start of embed from last_n_tokens + _embed.InsertRange(0, _last_n_tokens.GetRange(_n_ctx - n_left / 2 - _embed.Count, _embed.Count)); + + // stop saving session if we run out of context + _path_session = ""; + + // Console.WriteLine("\n---\n"); + // Console.Write("resetting: '"); + // for (int i = 0; i < embed.Count; i++) { + // Console.Write(llama_token_to_str(ctx, embed[i])); + // } + // Console.WriteLine("'\n"); + // Console.WriteLine("\n---\n"); } - } - catch (KeyNotFoundException) - { - if (_verbose) + + // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) + // REVIEW + if (_n_session_consumed < _session_tokens.Count) { - Logger.Default.Warn("Llama._create_completion: cache miss"); + int i = 0; + for (; i < _embed.Count; i++) + { + if (!_embed[i].Equals(_session_tokens[_n_session_consumed])) + { + _session_tokens.RemoveRange(_n_session_consumed, _session_tokens.Count - _n_session_consumed); + break; + } + + _n_past++; + _n_session_consumed++; + + if (_n_session_consumed >= _session_tokens.Count) + { + i++; + break; + } + } + + if (i > 0) + { + _embed.RemoveRange(0, i); + } } - } - } - string finishReason = "length"; - int multibyteFix = 0; - bool reset = true; - List tokens = new(promptTokens); - if (reset && _eval_tokens.Count > 0) - { - int longest_prefix = 0; - foreach (var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count - 1), (x, y) => (x, y))) - { - if (a == b) + // evaluate tokens in batches + // embed is typically prepared beforehand to fit within a batch, but not always + for (int i = 0; i < _embed.Count; i += _params.n_batch) { - longest_prefix += 1; + int n_eval = _embed.Count - i; + + if (n_eval > _params.n_batch) + { + n_eval = _params.n_batch; + } + + var array = _embed.GetRange(i, n_eval).ToArray(); + if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0) + { + Logger.Default.Error($"Failed to eval"); + throw new RuntimeError("Failed to eval"); + } + + _n_past += n_eval; } - else + + if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session)) { - break; + _session_tokens.AddRange(_embed); + _n_session_consumed = _session_tokens.Count; } } - if (longest_prefix > 0) + + _embed.Clear(); + + if (_embed_inp.Count <= _n_consumed && !_is_interacting) { - if (_verbose) + var temp = _params.temp; + var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k; + var top_p = _params.top_p; + var tfs_z = _params.tfs_z; + var typical_p = _params.typical_p; + var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n; + var repeat_penalty = _params.repeat_penalty; + var alpha_presence = _params.presence_penalty; + var alpha_frequency = _params.frequency_penalty; + var mirostat = _params.mirostat; + var mirostat_tau = _params.mirostat_tau; + var mirostat_eta = _params.mirostat_eta; + var penalize_nl = _params.penalize_nl; + + // optionally save the session on first sample (for faster prompt loading next time) + if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session) { - Logger.Default.Info("Llama.generate: prefix-match hit"); + _need_to_save_session = false; + NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count); } - reset = false; - tokens = tokens.Skip(longest_prefix).ToList(); - for (int i = 0; i < _eval_tokens.Count - longest_prefix; i++) + + llama_token id = 0; + { - _eval_tokens.Dequeue(); - if (_eval_logits.Count > 0) + var n_vocab = NativeApi.llama_n_vocab(_ctx); + var logits = Utils.llama_get_logits(_ctx, n_vocab); + + // Apply params.logit_bias map + foreach (KeyValuePair it in _params.logit_bias) { - _eval_logits.Dequeue(); + logits[it.Key] += it.Value; } - } - } - } - if (reset) - { - Reset(); - } - //foreach (var token in Generate(promptTokens, top_k, top_p, temperature, frequency_penalty, presence_penalty, repeat_penalty)) - string allText = ""; - while (true) - { - Eval(tokens); - var token = Sample(top_k, top_p, temperature, repeat_penalty, frequency_penalty, presence_penalty); - tokens.Clear(); - tokens.Add(token); - if (token == NativeApi.llama_token_eos()) - { - text = DeTokenize(completionTokens); - finishReason = "stop"; - break; - } + var candidates = new List(); + candidates.Capacity = n_vocab; + for (llama_token token_id = 0; token_id < n_vocab; token_id++) + { + candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f)); + } + + LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false); + + // Apply penalties + float nl_logit = logits[NativeApi.llama_token_nl()]; + var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx); + SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p, + _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(), + (ulong)last_n_repeat, repeat_penalty); + SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p, + _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(), + (ulong)last_n_repeat, alpha_frequency, alpha_presence); + if (!penalize_nl) + { + logits[NativeApi.llama_token_nl()] = nl_logit; + } - completionTokens.Add(token); + if (temp <= 0) + { + // Greedy sampling + id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p); + } + else + { + if (mirostat == 1) + { + float mirostat_mu = 2.0f * mirostat_tau; + const int mirostat_m = 100; + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, mirostat_mu); + } + else if (mirostat == 2) + { + float mirostat_mu = 2.0f * mirostat_tau; + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_mu); + } + else + { + // Temperature sampling + SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1); + SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1); + SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1); + SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1); + SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp); + id = SamplingApi.llama_sample_token(_ctx, candidates_p); + } + } - allText = DeTokenize(completionTokens); + _last_n_tokens.RemoveAt(0); + _last_n_tokens.Add(id); + } - int cut = Math.Min(3, allText.Length); - for(int i = allText.Length - cut; i < allText.Length; i++) - { - var c = (int)allText[i]; - int k = cut - i; - foreach(var (num, pattern) in _numAndPatterns) + // replace end of text token with newline token when in interactive mode + if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct) { - if(num > k && (pattern & c) == pattern) + id = _llama_token_newline[0]; + if (_params.antiprompt.Count != 0) { - multibyteFix = num - k; + // tokenize and inject first reverse prompt + var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false); + _embed_inp.AddRange(first_antiprompt); } } - } - if(multibyteFix > 0) - { - multibyteFix--; - continue; - } + // add it to the context + _embed.Add(id); - var anyStop = stop.Where(s => allText.Contains(s)); - if(anyStop.Count() > 0) - { - var firstStop = anyStop.First(); - text = allText.Substring(0, allText.IndexOf(firstStop)); - finishReason = "stop"; - break; - } + // echo this to console + _input_echo = true; - var start = returnedCharacters; - int longest = 0; - foreach (var s in stop) + // decrement remaining sampling budget + _n_remain--; + } + else { - for (int i = s.Length; i > 0; i--) + // Assuming that the necessary variables have been defined and initialized, + // the C# equivalent code could be: + + while (_embed_inp.Count > _n_consumed) { - if (allText.EndsWith(s.Substring(0, i))) + _embed.Add(_embed_inp[_n_consumed]); + _last_n_tokens.RemoveAt(0); + _last_n_tokens.Add(_embed_inp[_n_consumed]); + _n_consumed++; + if (_embed.Count >= _params.n_batch) { - if (i > longest) - { - longest = i; - } break; } } } - text = allText.Substring(0, allText.Length - longest); - returnedCharacters += text.Skip(start).Count(); - yield return new CompletionChunk(completionId, "text_completion", created, _model_path, new CompletionChoice[] - { - new CompletionChoice(text.Substring(start), 0, null, finishReason) - }); - } - - if (_cache is not null) - { - if (_verbose) - { - Logger.Default.Info("Llama._create_completion: cache save"); - } - _cache[promptTokens.Concat(completionTokens).ToArray()] = SaveState(); - } - - string textStr = text; - if (echo) - { - textStr = prompt + textStr; - } - if(suffix is not null) - { - textStr = textStr + suffix; - } - CompletionLogprobs? logProbs = null; - if (logprobs != -1) - { - int textOffset = 0; - List textOffsets = new(); - List tokenLogprobs = new(); - List tokenStrs = new(); - List> topLogprobs = new(); - - var allTokens = promptTokens.Concat(completionTokens).ToArray(); - var allTokenStrs = allTokens.Select(t => DeTokenize(new[] { t })); - var allLogProbs = _eval_logits.Select(row => LogitsToLogprobs(row)); - - foreach (var (token, tokenStr, logProbsToken) in allTokens.Zip(allTokenStrs, (x, y) => (x, y)) - .Zip(allLogProbs, (x, y) => (x.x, x.y, y))) + if (_input_echo) { - textOffsets.Add(textOffset); - textOffset += tokenStr.Length; - tokenStrs.Add(tokenStr); - var sortedLogprobs = logProbsToken.Zip(Enumerable.Range(0, logProbsToken.Count()), (x, y) => (x, y)) - .OrderByDescending(x => x.x).ToList(); - tokenLogprobs.Add(sortedLogprobs[token].x); - var topLogprob = sortedLogprobs.Take(logprobs).ToDictionary(t => DeTokenize(new[] { t.y }), t => t.x); - topLogprob[tokenStr] = sortedLogprobs[token].x; - topLogprobs.Add(topLogprob); + foreach (var id in _embed) + { + yield return Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); + } } - logProbs = new(textOffsets.ToArray(), tokenLogprobs.ToArray(), tokenStrs.ToArray(), topLogprobs.ToArray()); - } - - if (_verbose) - { - NativeApi.llama_print_timings(_ctx); - } - } - - /// - /// Generate text from a prompt and yield return the result. - /// - /// The prompt to generate text from. - /// A suffix to append to the generated text. If None, no suffix is appended. - /// The maximum number of tokens to generate. - /// The temperature to use for sampling. - /// The top-p value to use for sampling. - /// The number of logprobs to return. If None, no logprobs are returned. - /// Whether to echo the prompt. - /// A list of strings to stop generation when encountered. - /// - /// - /// The penalty to apply to repeated tokens. - /// The top-k value to use for sampling. - /// - public IEnumerable CreateCompletion(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, - float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, - float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) - { - return CreateCompletionInternal(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, - frequency_penalty, presence_penalty, repeat_penalty, top_k); - } - - /// - /// Generate text from a prompt and yield return the result. - /// - /// The prompt to generate text from. - /// A suffix to append to the generated text. If None, no suffix is appended. - /// The maximum number of tokens to generate. - /// The temperature to use for sampling. - /// The top-p value to use for sampling. - /// The number of logprobs to return. If None, no logprobs are returned. - /// Whether to echo the prompt. - /// A list of strings to stop generation when encountered. - /// - /// - /// The penalty to apply to repeated tokens. - /// The top-k value to use for sampling. - /// - public IEnumerable Call(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, - float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, - float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) - { - return CreateCompletion(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, - frequency_penalty, presence_penalty, repeat_penalty, top_k); - } - - private ChatCompletion ConvertTextCompletionToChat(Completion completion) - { - return new ChatCompletion($"chat{completion.Id}", "chat.completion", completion.Created, completion.Model, - new[] { new ChatCompletionChoice(0, new ChatCompletionMessage(ChatRole.Assistant, completion.Choices[0].Text), - completion.Choices[0].FinishReason) }, completion.Usage); - } - - private IEnumerable ConvertTextCompletionChunksToChat(IEnumerable chunks) - { - bool isFirst = true; - foreach(var chunk in chunks) - { - if(isFirst) + if (_params.interactive && _embed_inp.Count <= _n_consumed) { - yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, - new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta("assistant", null), null) }); - isFirst = false; - } - yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, - new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta(null, chunk.Choices[0].Text), - chunk.Choices[0].FinishReason) }); - } - } - - /// - /// Generate a chat completion from a list of messages and yield return the result. - /// - /// A list of messages to generate a response for. - /// The temperature to use for sampling. - /// The top-p value to use for sampling. - /// The top-k value to use for sampling. - /// A list of strings to stop generation when encountered. - /// The maximum number of tokens to generate. - /// - /// - /// The penalty to apply to repeated tokens. - /// - public IEnumerable CreateChatCompletion(IEnumerable messages, float temperature = .2f, float top_p = .95f, - int top_k = 40, string[]? stop = null, int max_tokens = 256, float presence_penalty = .0f, float frequency_penalty = .0f, - float repeat_penalty = 1.1f) - { - if (stop is null) - { - stop = new string[0]; - } - string GetRole(ChatCompletionMessage message) - { - return message.Role == ChatRole.Human ? "Human" : "Assistant"; - } - string chatHistory = string.Join("", messages.Select(m => $"### {GetRole(m)}:{m.Content}")); - var prompt = chatHistory + "### Assistant:"; - prompt = prompt.Substring(Math.Max(0, prompt.Length - max_tokens)); - var promptStop = new[] { "### Assistant:", "### Human:" }.Concat(stop).ToArray(); - var completion = Call(prompt, stop: promptStop, temperature: temperature, top_p: top_p, top_k: top_k, max_tokens: max_tokens, - repeat_penalty: repeat_penalty, presence_penalty: presence_penalty, frequency_penalty: frequency_penalty); - return ConvertTextCompletionChunksToChat(completion); - } + if (_params.antiprompt.Count > 0) + { + string last_output = ""; + foreach (var id in _last_n_tokens) + { + last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id)); + } - public LLamaState SaveState() - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - ulong stateSize = NativeApi.llama_get_state_size(_ctx); - byte[] llamaState = new byte[stateSize]; - ulong nBytes = NativeApi.llama_copy_state_data(_ctx, llamaState); - if(nBytes > stateSize) - { - throw new RuntimeError("Failed to copy llama state data"); - } - byte[] llamaStateCompact = new byte[nBytes]; - llamaState.Take((int)nBytes).ToArray().CopyTo(llamaStateCompact, 0); - if (_verbose) - { - Logger.Default.Info($"Llama.save_state: saving {nBytes} bytes of llama state"); - } - return new LLamaState(new Queue(_eval_tokens), new Queue(_eval_logits), - llamaStateCompact, (int)nBytes); - } + _is_antiprompt = false; + foreach (var antiprompt in _params.antiprompt) + { + if (last_output.EndsWith(antiprompt)) + { + _is_interacting = true; + _is_antiprompt = true; + break; + } + } + } - public void LoadState(LLamaState state) - { - Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); - _eval_tokens = new Queue(state.EvalTokens); - _eval_logits = new Queue(state.EvalLogits); - if(NativeApi.llama_set_state_data(_ctx, state.State) != (ulong)state.Size) - { - throw new RuntimeError($"Failed to set llama state data"); - } - } + if(_n_past > 0 && _is_interacting) + { + _input_echo = false; + break; + } - private static IEnumerable LogitsToLogprobs(IEnumerable logits) - { - var exps = logits.Select(x => (float)Math.Exp(x)); - var sumExps = exps.Sum(); - return exps.Select(x => (float)Math.Log(x / sumExps)); - } + if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos()) + { + if (_params.instruct) { + _is_interacting = true; + } else + { + Logger.Default.Info(" [end of text]"); + } + } - internal static int LongestTokenPrefix(IEnumerable a, IEnumerable b) - { - int longestPrefix = 0; - foreach(var (x, y) in a.Zip(b, (x, y) => (x, y))) - { - if(x == y) - { - longestPrefix++; - } - else - { - break; + if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1) { + _n_remain = _params.n_predict; + _is_interacting = true; + } } } - return longestPrefix; } public void Dispose() diff --git a/LLama/LLamaModelV1.cs b/LLama/LLamaModelV1.cs new file mode 100644 index 00000000..c984a63d --- /dev/null +++ b/LLama/LLamaModelV1.cs @@ -0,0 +1,837 @@ +using LLama.Exceptions; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using LLama.Types; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; +using System.Collections; + +namespace LLama +{ + using llama_token = Int32; + /// + /// High-level Wrapper of a llama.cpp model for inference. Note that it's more recommended to use `LLamaModel`. + /// This class may be removed in the future. However, if all you want is to get the embeddings, then using `LLamaModelV1` + /// is ok now. + /// + [Obsolete] + public class LLamaModelV1: IDisposable + { + private string _model_path; + LLamaContextParams _params; + private int _n_threads; + private int _n_batch; + private int _last_n_tokens_size; + private string? _lora_base; + private string? _lora_path; + private bool _verbose; + + private Queue _eval_tokens; + private Queue _eval_logits; + private LLamaCache? _cache; + private SafeLLamaContextHandle _ctx; + + private static readonly (int, int)[] _numAndPatterns = new (int, int)[] { (2, 192), (3, 224), (4, 240) }; + + /// + /// Load a llama.cpp model from the path. + /// + /// Note that the API is still unstable. The order of them is likely to + /// be changed in the future. It's recommened to specify the parameter name when + /// building your app. We use the cpp style parameter names here because it introduces + /// convenience for searching the docs. + /// Path to the model. + /// Maximum context size. + /// Number of parts to split the model into. If -1, the number of parts is automatically determined. + /// Random seed. 0 for random. + /// Use half-precision for key/value cache. + /// Return logits for all tokens, not just the last token. + /// Only load the vocabulary no weights. + /// Use mmap if possible. + /// Force the system to keep the model in RAM. + /// Embedding mode only. + /// Number of threads to use. If is not specified, the number of threads is automatically determined. + /// Maximum number of prompt tokens to batch together when calling llama_eval. + /// Maximum number of tokens to keep in the last_n_tokens deque. + /// Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. + /// Path to a LoRA file to apply to the model. + /// Print verbose output to stderr. + public LLamaModelV1(string model_path, int n_ctx = 512, int n_parts = -1, int seed = 1337, + bool f16_kv = true, bool logits_all = false, bool vocab_only = false, bool use_mmap = true, + bool use_mlock = false, bool embedding = false, int n_threads = -1, int n_batch = 512, + int last_n_tokens_size = 64, string? lora_base = null, string? lora_path = null, bool verbose = true) + { + _verbose = verbose; + _model_path = model_path; + + _params = NativeApi.llama_context_default_params(); + _params.n_ctx = n_ctx; + _params.n_parts = n_parts; + _params.seed = seed; + _params.f16_kv = f16_kv; + _params.logits_all = logits_all; + _params.vocab_only = vocab_only; + _params.use_mmap = lora_path is null ? use_mmap : false; + _params.use_mlock = use_mlock; + _params.embedding = embedding; + + _last_n_tokens_size = last_n_tokens_size; + _n_batch = Math.Min(n_ctx, n_batch); + + _eval_tokens = new Queue(capacity: n_ctx); + _eval_logits = new Queue(logits_all ? n_ctx : 1); + + _cache = null; + + _n_threads = n_threads; + if(_n_threads == -1) + { + _n_threads = Math.Max(Environment.ProcessorCount / 2, 1); + } + + _lora_base = lora_base; + _lora_path = lora_path; + + if(!File.Exists(model_path) && !Directory.Exists(model_path)) + { + throw new FileNotFoundException($"Model path does not exist: {model_path}"); + } + + // Move from heap to stack to prevent the moving. + _ctx = new SafeLLamaContextHandle(NativeApi.llama_init_from_file(Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(model_path)), _params)); + + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + + if(_lora_path is not null) + { + if(NativeApi.llama_apply_lora_from_file(_ctx, lora_path, lora_base, _n_threads) != 0) + { + throw new RuntimeError($"Failed to apply LoRA from lora path: {_lora_path} to base path: {_lora_base}"); + } + } + + if (_verbose) + { + Logger.Default.Info(Utils.PtrToStringUTF8(NativeApi.llama_print_system_info())); + } + } + + public LLamaModelV1(LLamaModelV1 other) + { + _ctx = other._ctx; + _model_path = other._model_path; + _params = other._params; + _last_n_tokens_size = other._last_n_tokens_size; + _n_threads = other._n_threads; + _n_batch = other._n_batch; + _verbose = other._verbose; + _lora_base = other._lora_base; + _lora_path = other._lora_path; + _eval_logits = new Queue(other._eval_logits); + _eval_tokens = new Queue(other._eval_tokens); + } + + /// + /// Tokenize a string. + /// + /// The utf-8 encoded string to tokenize. + /// A list of tokens. + /// If the tokenization failed. + public List Tokenize(string text) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var n_ctx = NativeApi.llama_n_ctx(_ctx); + var tokens = new llama_token[n_ctx]; + var n_tokens = NativeApi.llama_tokenize(_ctx, text, tokens, n_ctx, true); + if(n_tokens < 0) + { + throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}"); + } + return tokens.Take(n_tokens).ToList(); + } + + /// + /// Detokenize a list of tokens. + /// + /// The list of tokens to detokenize. + /// The detokenized string. + public string DeTokenize(IEnumerable tokens) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + string output = ""; + foreach(var token in tokens) + { + output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)); + } + return output; + } + + public string DeTokenize(llama_token token) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + return Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)) ?? ""; + } + + /// + /// Set the cache. + /// + /// The cache to set. + public void SetCache(LLamaCache? cache) + { + _cache = cache; + } + + /// + /// Reset the model state. + /// + public void Reset() + { + _eval_tokens.Clear(); + _eval_logits.Clear(); + } + + /// + /// Evaluate a list of tokens. + /// + /// The list of tokens to evaluate. + /// + public unsafe void Eval(List tokens) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var n_ctx = NativeApi.llama_n_ctx(_ctx); + for(int i = 0; i < tokens.Count; i += _n_batch) + { + var batch = tokens.Take(Math.Min(tokens.Count, i + _n_batch)).Skip(i); + llama_token n_past = Math.Min(n_ctx - batch.Count(), _eval_tokens.Count); + llama_token n_tokens = batch.Count(); + llama_token return_code = NativeApi.llama_eval( + ctx: _ctx, + tokens: batch.ToArray(), + n_tokens: n_tokens, + n_past: n_past, + n_threads: _n_threads + ); + if(return_code != 0) + { + throw new RuntimeError($"llama_eval returned {return_code}"); + } + foreach(var b in batch) + { + _eval_tokens.Enqueue(b); + } + int rows = _params.logits_all ? n_tokens : 1; + llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); + var cols = n_vocab; + var logits_view = NativeApi.llama_get_logits(_ctx); + for(int j = 0; j < rows; j++) + { + float[] logit = new float[cols]; + for(int k = 0; k < cols; k++) + { + logit[k] = logits_view[j * cols + k]; + } + _eval_logits.Enqueue(logit); + } + } + } + + private llama_token SampleInternal(llama_token[] last_n_tokens_data, int last_n_tokens_size, int top_k, + float top_p, float temp, float repeat_penalty, float frequency_penalty, float presence_penalty) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + Debug.Assert(_eval_logits.Count > 0); + llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); + var logits = _eval_logits.Last(); + LLamaTokenData[] data = new LLamaTokenData[n_vocab]; + for(int i = 0; i < n_vocab; i++) + { + data[i] = new LLamaTokenData(i, logits[i], .0f); + } + ulong size = (ulong)n_vocab; + bool sorted = false; + LLamaTokenDataArray candidates = new(data, size, sorted); + SamplingApi.llama_sample_repetition_penalty(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, + repeat_penalty); + //SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, + // frequency_penalty, presence_penalty); + if(temp == .0f) + { + return SamplingApi.llama_sample_token_greedy(_ctx, candidates); + } + else + { + SamplingApi.llama_sample_top_k(_ctx, candidates, top_k, 1); + SamplingApi.llama_sample_tail_free(_ctx, candidates, 1.0f, 1); + SamplingApi.llama_sample_typical(_ctx, candidates, 1.0f, 1); + SamplingApi.llama_sample_top_p(_ctx, candidates, top_p, 1); + SamplingApi.llama_sample_temperature(_ctx, candidates, temp); + return SamplingApi.llama_sample_token(_ctx, candidates); + } + } + + /// + /// Sample a token from the model. + /// + /// The top-k sampling parameter. + /// The top-p sampling parameter. + /// The temperature parameter. + /// The repeat penalty parameter. + /// + /// + /// The sampled token. + public llama_token Sample(int top_k, float top_p, float temp, float repeat_penalty, float frequency_penalty = .0f, + float presence_penalty = .0f) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var last_n_tokens_data = Enumerable.Repeat(0, Math.Max(0, _last_n_tokens_size - _eval_tokens.Count)); + last_n_tokens_data = last_n_tokens_data.Concat(_eval_tokens.ToList() + .Skip(Math.Max(0, _eval_tokens.Count - _last_n_tokens_size))); + llama_token[] tokens_data = new llama_token[_last_n_tokens_size]; + int i = 0; + foreach(var data in last_n_tokens_data) + { + if(i < _last_n_tokens_size) + { + tokens_data[i++] = data; + } + else + { + break; + } + } + return SampleInternal(tokens_data, _last_n_tokens_size, top_k, top_p, temp, repeat_penalty, frequency_penalty, presence_penalty); + } + + /// + /// Create a generator of tokens from a prompt. + /// + /// + /// Examples: + /// var llama = new LlamaModel("models/ggml-7b.bin") + /// var tokens = llama.Tokenize(b"Hello, world!") + /// foreach(var token in llama.Generate(tokens, top_k:40, top_p:0.95, temp:1.0, repeat_penalty:1.1)){ + /// Console.WriteLine(llama.DeTokenize(new []{token})); + /// } + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public IEnumerable Generate(IEnumerable tokens, int top_k, float top_p, float temp, + float repeat_penalty, float frequency_penalty = .0f, float presence_penalty = .0f, bool reset = true) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + if(reset && _eval_tokens.Count > 0) + { + int longest_prefix = 0; + foreach(var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count() - 1), (x, y) => (x, y))) + { + if(a == b) + { + longest_prefix += 1; + } + else + { + break; + } + } + if(longest_prefix > 0) + { + if (_verbose) + { + Logger.Default.Info("Llama.generate: prefix-match hit"); + } + reset = false; + tokens = tokens.Skip(longest_prefix); + for(int i = 0; i < _eval_tokens.Count - longest_prefix; i++) + { + _eval_tokens.Dequeue(); + if(_eval_logits.Count > 0) + { + _eval_logits.Dequeue(); + } + } + } + } + + if (reset) + { + Reset(); + } + + while (true) + { + Eval(tokens.ToList()); + var token = Sample(top_k, top_p, temp, frequency_penalty, presence_penalty, repeat_penalty); + yield return token; + // TODO(Rinne): verify if the implementation is correct. + } + } + + /// + /// Embed a string. + /// + /// The utf-8 encoded string to embed. + /// An embedding object. + /// + public unsafe Embedding CreateEmbedding(string input) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + if (!_params.embedding) + { + throw new RuntimeError("Llama model must be created with embedding=True to call this method"); + } + + if (_verbose) + { + NativeApi.llama_reset_timings(_ctx); + } + + var tokens = Tokenize(input); + Reset(); + Eval(tokens); + int n_tokens = tokens.Count; + var embeddingPtr = NativeApi.llama_get_embeddings(_ctx); + int cnt = NativeApi.llama_n_embd(_ctx); + float[] embedding = new float[cnt]; + for(int i = 0; i < cnt; i++) + { + embedding[i] = embeddingPtr[i]; + } + + if (_verbose) + { + NativeApi.llama_print_timings(_ctx); + } + + return new Embedding("list", _model_path, new[] { new EmbeddingData(0, "embedding", embedding) }, + new EmbeddingUsage(n_tokens, n_tokens)); + } + + public float[] Embed(string input) + { + return CreateEmbedding(input).Data[0].Embedding; + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// IEnumerable of Completion and CompletionChunk + /// + private IEnumerable CreateCompletionInternal(string prompt, string?suffix = null, int max_tokens = 16, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + string completionId = $"cmpl-{Guid.NewGuid()}"; + var created = DateTime.Now.Millisecond; + List completionTokens = new List(); + + var promptTokens = Tokenize($" {prompt}"); + string text = ""; + int returnedCharacters = 0; + if(stop is null) + { + stop = new string[0]; + } + + if (_verbose) + { + NativeApi.llama_reset_timings(_ctx); + } + + if(promptTokens.Count + max_tokens > NativeApi.llama_n_ctx(_ctx)) + { + throw new ArgumentException($"Requested tokens exceed context window of {NativeApi.llama_n_ctx(_ctx)}"); + } + + if(logprobs != -1 && !_params.logits_all) + { + throw new ArgumentException("logprobs is not supported for models created with logits_all=False"); + } + + if(_cache is not null) + { + try + { + // TODO(Rinne): revise it since it will compare reference instead of elements. + var cacheItem = _cache[promptTokens.ToArray()]; + var cachePrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); + var evalPrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); + if(cachePrefixLen > evalPrefixLen) + { + LoadState(cacheItem); + if (_verbose) + { + Logger.Default.Info("Llama._create_completion: cache hit"); + } + } + } + catch (KeyNotFoundException) + { + if (_verbose) + { + Logger.Default.Warn("Llama._create_completion: cache miss"); + } + } + } + + string finishReason = "length"; + int multibyteFix = 0; + bool reset = true; + List tokens = new(promptTokens); + if (reset && _eval_tokens.Count > 0) + { + int longest_prefix = 0; + foreach (var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count - 1), (x, y) => (x, y))) + { + if (a == b) + { + longest_prefix += 1; + } + else + { + break; + } + } + if (longest_prefix > 0) + { + if (_verbose) + { + Logger.Default.Info("Llama.generate: prefix-match hit"); + } + reset = false; + tokens = tokens.Skip(longest_prefix).ToList(); + for (int i = 0; i < _eval_tokens.Count - longest_prefix; i++) + { + _eval_tokens.Dequeue(); + if (_eval_logits.Count > 0) + { + _eval_logits.Dequeue(); + } + } + } + } + + if (reset) + { + Reset(); + } + //foreach (var token in Generate(promptTokens, top_k, top_p, temperature, frequency_penalty, presence_penalty, repeat_penalty)) + string allText = ""; + while (true) + { + Eval(tokens); + var token = Sample(top_k, top_p, temperature, repeat_penalty, frequency_penalty, presence_penalty); + tokens.Clear(); + tokens.Add(token); + if (token == NativeApi.llama_token_eos()) + { + text = DeTokenize(completionTokens); + finishReason = "stop"; + break; + } + + completionTokens.Add(token); + + allText = DeTokenize(completionTokens); + + int cut = Math.Min(3, allText.Length); + for(int i = allText.Length - cut; i < allText.Length; i++) + { + var c = (int)allText[i]; + int k = cut - i; + foreach(var (num, pattern) in _numAndPatterns) + { + if(num > k && (pattern & c) == pattern) + { + multibyteFix = num - k; + } + } + } + + if(multibyteFix > 0) + { + multibyteFix--; + continue; + } + + var anyStop = stop.Where(s => allText.Contains(s)); + if(anyStop.Count() > 0) + { + var firstStop = anyStop.First(); + text = allText.Substring(0, allText.IndexOf(firstStop)); + finishReason = "stop"; + break; + } + + var start = returnedCharacters; + int longest = 0; + foreach (var s in stop) + { + for (int i = s.Length; i > 0; i--) + { + if (allText.EndsWith(s.Substring(0, i))) + { + if (i > longest) + { + longest = i; + } + break; + } + } + } + text = allText.Substring(0, allText.Length - longest); + returnedCharacters += text.Skip(start).Count(); + yield return new CompletionChunk(completionId, "text_completion", created, _model_path, new CompletionChoice[] + { + new CompletionChoice(text.Substring(start), 0, null, finishReason) + }); + } + + if (_cache is not null) + { + if (_verbose) + { + Logger.Default.Info("Llama._create_completion: cache save"); + } + _cache[promptTokens.Concat(completionTokens).ToArray()] = SaveState(); + } + + string textStr = text; + if (echo) + { + textStr = prompt + textStr; + } + if(suffix is not null) + { + textStr = textStr + suffix; + } + + CompletionLogprobs? logProbs = null; + if (logprobs != -1) + { + int textOffset = 0; + List textOffsets = new(); + List tokenLogprobs = new(); + List tokenStrs = new(); + List> topLogprobs = new(); + + var allTokens = promptTokens.Concat(completionTokens).ToArray(); + var allTokenStrs = allTokens.Select(t => DeTokenize(new[] { t })); + var allLogProbs = _eval_logits.Select(row => LogitsToLogprobs(row)); + + foreach (var (token, tokenStr, logProbsToken) in allTokens.Zip(allTokenStrs, (x, y) => (x, y)) + .Zip(allLogProbs, (x, y) => (x.x, x.y, y))) + { + textOffsets.Add(textOffset); + textOffset += tokenStr.Length; + tokenStrs.Add(tokenStr); + var sortedLogprobs = logProbsToken.Zip(Enumerable.Range(0, logProbsToken.Count()), (x, y) => (x, y)) + .OrderByDescending(x => x.x).ToList(); + tokenLogprobs.Add(sortedLogprobs[token].x); + var topLogprob = sortedLogprobs.Take(logprobs).ToDictionary(t => DeTokenize(new[] { t.y }), t => t.x); + topLogprob[tokenStr] = sortedLogprobs[token].x; + topLogprobs.Add(topLogprob); + } + + logProbs = new(textOffsets.ToArray(), tokenLogprobs.ToArray(), tokenStrs.ToArray(), topLogprobs.ToArray()); + } + + if (_verbose) + { + NativeApi.llama_print_timings(_ctx); + } + } + + /// + /// Generate text from a prompt and yield return the result. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public IEnumerable CreateCompletion(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + return CreateCompletionInternal(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k); + } + + /// + /// Generate text from a prompt and yield return the result. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public IEnumerable Call(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + return CreateCompletion(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k); + } + + private ChatCompletion ConvertTextCompletionToChat(Completion completion) + { + return new ChatCompletion($"chat{completion.Id}", "chat.completion", completion.Created, completion.Model, + new[] { new ChatCompletionChoice(0, new ChatCompletionMessage(ChatRole.Assistant, completion.Choices[0].Text), + completion.Choices[0].FinishReason) }, completion.Usage); + } + + private IEnumerable ConvertTextCompletionChunksToChat(IEnumerable chunks) + { + bool isFirst = true; + foreach(var chunk in chunks) + { + if(isFirst) + { + yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, + new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta("assistant", null), null) }); + isFirst = false; + } + yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, + new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta(null, chunk.Choices[0].Text), + chunk.Choices[0].FinishReason) }); + } + } + + /// + /// Generate a chat completion from a list of messages and yield return the result. + /// + /// A list of messages to generate a response for. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The top-k value to use for sampling. + /// A list of strings to stop generation when encountered. + /// The maximum number of tokens to generate. + /// + /// + /// The penalty to apply to repeated tokens. + /// + public IEnumerable CreateChatCompletion(IEnumerable messages, float temperature = .2f, float top_p = .95f, + int top_k = 40, string[]? stop = null, int max_tokens = 256, float presence_penalty = .0f, float frequency_penalty = .0f, + float repeat_penalty = 1.1f) + { + if (stop is null) + { + stop = new string[0]; + } + string GetRole(ChatCompletionMessage message) + { + return message.Role == ChatRole.Human ? "Human" : "Assistant"; + } + string chatHistory = string.Join("", messages.Select(m => $"### {GetRole(m)}:{m.Content}")); + var prompt = chatHistory + "### Assistant:"; + prompt = prompt.Substring(Math.Max(0, prompt.Length - max_tokens)); + var promptStop = new[] { "### Assistant:", "### Human:" }.Concat(stop).ToArray(); + var completion = Call(prompt, stop: promptStop, temperature: temperature, top_p: top_p, top_k: top_k, max_tokens: max_tokens, + repeat_penalty: repeat_penalty, presence_penalty: presence_penalty, frequency_penalty: frequency_penalty); + return ConvertTextCompletionChunksToChat(completion); + } + + public LLamaState SaveState() + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + ulong stateSize = NativeApi.llama_get_state_size(_ctx); + byte[] llamaState = new byte[stateSize]; + ulong nBytes = NativeApi.llama_copy_state_data(_ctx, llamaState); + if(nBytes > stateSize) + { + throw new RuntimeError("Failed to copy llama state data"); + } + byte[] llamaStateCompact = new byte[nBytes]; + llamaState.Take((int)nBytes).ToArray().CopyTo(llamaStateCompact, 0); + if (_verbose) + { + Logger.Default.Info($"Llama.save_state: saving {nBytes} bytes of llama state"); + } + return new LLamaState(new Queue(_eval_tokens), new Queue(_eval_logits), + llamaStateCompact, (int)nBytes); + } + + public void LoadState(LLamaState state) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + _eval_tokens = new Queue(state.EvalTokens); + _eval_logits = new Queue(state.EvalLogits); + if(NativeApi.llama_set_state_data(_ctx, state.State) != (ulong)state.Size) + { + throw new RuntimeError($"Failed to set llama state data"); + } + } + + private static IEnumerable LogitsToLogprobs(IEnumerable logits) + { + var exps = logits.Select(x => (float)Math.Exp(x)); + var sumExps = exps.Sum(); + return exps.Select(x => (float)Math.Log(x / sumExps)); + } + + internal static int LongestTokenPrefix(IEnumerable a, IEnumerable b) + { + int longestPrefix = 0; + foreach(var (x, y) in a.Zip(b, (x, y) => (x, y))) + { + if(x == y) + { + longestPrefix++; + } + else + { + break; + } + } + return longestPrefix; + } + + public void Dispose() + { + _ctx.Dispose(); + } + } +}