refactor: LLamaModel and LLamaExecutor.

LLama/Abstractions/Params/ModelParams.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions.Params
+{
+    public class ModelParams
+    {
+        /// <summary>
+        /// Model context size (n_ctx)
+        /// </summary>
+        public int ContextSize { get; set; } = 512;
+
+        /// <summary>
+        /// Number of layers to run in VRAM / GPU memory (n_gpu_layers)
+        /// </summary>
+        public int GpuLayerCount { get; set; } = 20;
+        /// <summary>
+        /// Seed for the random number generator (seed)
+        /// </summary>
+        public int Seed { get; set; } = 1686349486;
+        /// <summary>
+        /// Use f16 instead of f32 for memory kv (memory_f16)
+        /// </summary>
+        public bool UseFp16Memory { get; set; } = true;
+        /// <summary>
+        /// Use mmap for faster loads (use_mmap)
+        /// </summary>
+        public bool UseMemorymap { get; set; } = true;
+        /// <summary>
+        /// Use mlock to keep model in memory (use_mlock)
+        /// </summary>
+        public bool UseMemoryLock { get; set; } = false;
+        /// <summary>
+        /// Compute perplexity over the prompt (perplexity)
+        /// </summary>
+        public bool Perplexity { get; set; } = false;
+        /// <summary>
+        /// Model path (model)
+        /// </summary>
+        public string ModelPath { get; set; }
+        /// <summary>
+        /// lora adapter path (lora_adapter)
+        /// </summary>
+        public string LoraAdapter { get; set; } = string.Empty;
+        /// <summary>
+        /// base model path for the lora adapter (lora_base)
+        /// </summary>
+        public string LoraBase { get; set; } = string.Empty;
+        /// <summary>
+        /// Number of threads (-1 = autodetect) (n_threads)
+        /// </summary>
+        public int Threads { get; set; } = Math.Max(Environment.ProcessorCount / 2, 1);
+        /// <summary>
+        /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch)
+        /// </summary>
+        public int BatchSize { get; set; } = 512;
+
+        /// <summary>
+        /// Whether to convert eos to newline during the inference.
+        /// </summary>
+        public bool ConvertEosToNewLine { get; set; } = false;
+
+        /// <summary>
+        /// Whether to use embedding mode. (embedding) Note that if this is set to true, 
+        /// The LLamaModel won't produce text response anymore.
+        /// </summary>
+        public bool EmbeddingMode { get; set; } = false;
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="modelPath">The model path.</param>
+        /// <param name="contextSize">Model context size (n_ctx)</param>
+        /// <param name="gpuLayerCount">Number of layers to run in VRAM / GPU memory (n_gpu_layers)</param>
+        /// <param name="seed">Seed for the random number generator (seed)</param>
+        /// <param name="useFp16Memory">Whether to use f16 instead of f32 for memory kv (memory_f16)</param>
+        /// <param name="useMemorymap">Whether to use mmap for faster loads (use_mmap)</param>
+        /// <param name="useMemoryLock">Whether to use mlock to keep model in memory (use_mlock)</param>
+        /// <param name="perplexity">Thether to compute perplexity over the prompt (perplexity)</param>
+        /// <param name="loraAdapter">Lora adapter path (lora_adapter)</param>
+        /// <param name="loraBase">Base model path for the lora adapter (lora_base)</param>
+        /// <param name="threads">Number of threads (-1 = autodetect) (n_threads)</param>
+        /// <param name="batchSize">Batch size for prompt processing (must be >=32 to use BLAS) (n_batch)</param>
+        /// <param name="convertEosToNewLine">Whether to convert eos to newline during the inference.</param>
+        /// <param name="embeddingMode">Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore.</param>
+        public ModelParams(string modelPath, int contextSize = 512, int gpuLayerCount = 20, 
+                   int seed = 1337, bool useFp16Memory = true,
+                   bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false,
+                   string loraAdapter = "", string loraBase = "", int threads = -1, int batchSize = 512, 
+                   bool convertEosToNewLine = false, bool embeddingMode = false)
+        {
+            ContextSize = contextSize;
+            GpuLayerCount = gpuLayerCount;
+            Seed = seed;
+            UseFp16Memory = useFp16Memory;
+            UseMemorymap = useMemorymap;
+            UseMemoryLock = useMemoryLock;
+            Perplexity = perplexity;
+            ModelPath = modelPath;
+            LoraAdapter = loraAdapter;
+            LoraBase = loraBase;
+            Threads = threads == -1 ? Math.Max(Environment.ProcessorCount / 2, 1) : threads;
+            BatchSize = batchSize;
+            ConvertEosToNewLine = convertEosToNewLine;
+            EmbeddingMode = embeddingMode;
+        }
+    }
+}

LLama/Abstractions/Params/SessionParams.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions.Params
+{
+    using llama_token = Int32;
+    public class SessionParams
+    {
+        /// <summary>
+        /// number of tokens to keep from initial prompt
+        /// </summary>
+        public int TokensToKeep { get; set; } = 0;
+        /// <summary>
+        /// how many new tokens to predict (n_predict), set to -1 to inifinitely generate response
+        /// until it complete.
+        /// </summary>
+        public int ResponseTokensCount { get; set; } = -1;
+        /// <summary>
+        /// logit bias for specific tokens
+        /// </summary>
+        public Dictionary<llama_token, float>? LogitBias { get; set; } = null;
+        /// <summary>
+        /// path to file for saving/loading model eval state
+        /// </summary>
+        public string PathSession { get; set; } = string.Empty;
+        /// <summary>
+        /// string to suffix user inputs with
+        /// </summary>
+        public string InputSuffix { get; set; } = string.Empty;
+        /// <summary>
+        /// string to prefix user inputs with
+        /// </summary>
+        public string InputPrefix { get; set; } = string.Empty;
+        /// <summary>
+        ///  0 or lower to use vocab size
+        /// </summary>
+        public int TopK { get; set; } = 40;
+        /// <summary>
+        /// 1.0 = disabled
+        /// </summary>
+        public float TopP { get; set; } = 0.95f;
+        /// <summary>
+        /// 1.0 = disabled
+        /// </summary>
+        public float TfsZ { get; set; } = 1.0f;
+        /// <summary>
+        /// 1.0 = disabled
+        /// </summary>
+        public float TypicalP { get; set; } = 1.0f;
+        /// <summary>
+        /// 1.0 = disabled
+        /// </summary>
+        public float Temperature { get; set; } = 0.8f;
+        /// <summary>
+        /// 1.0 = disabled
+        /// </summary>
+        public float RepeatPenalty { get; set; } = 1.1f;
+        /// <summary>
+        /// last n tokens to penalize (0 = disable penalty, -1 = context size) (repeat_last_n)
+        /// </summary>
+        public int RepeatLastTokensCount { get; set; } = 64;
+        /// <summary>
+        /// frequency penalty coefficient
+        /// 0.0 = disabled
+        /// </summary>
+        public float FrequencyPenalty { get; set; } = .0f;
+        /// <summary>
+        /// presence penalty coefficient
+        /// 0.0 = disabled
+        /// </summary>
+        public float PresencePenalty { get; set; } = .0f;
+        /// <summary>
+        /// Mirostat uses tokens instead of words.
+        /// algorithm described in the paper https://arxiv.org/abs/2007.14966.
+        /// 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
+        /// </summary>
+        public MiroStateType Mirostat { get; set; } = MiroStateType.Disable;
+        /// <summary>
+        /// target entropy
+        /// </summary>
+        public float MirostatTau { get; set; } = 5.0f;
+        /// <summary>
+        /// learning rate
+        /// </summary>
+        public float MirostatEta { get; set; } = 0.1f;
+        /// <summary>
+        /// consider newlines as a repeatable token (penalize_nl)
+        /// </summary>
+        public bool PenalizeNL { get; set; } = true;
+    }
+
+    public enum MiroStateType
+    {
+        Disable = 0, 
+        MiroState = 1, 
+        MiroState2 = 2
+    }
+}

LLama/ChatSession.cs

@@ -1,53 +1,102 @@
-using LLama.Types;
+using LLama.Abstractions.Params;
+using LLama.Common;
+using LLama.Exceptions;
+using LLama.Native;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Reflection;
 using System.Text;
 
 namespace LLama
 {
-    public class ChatSession<T> where T: IChatModel
-    {
-        IChatModel _model;
-        List<ChatMessageRecord> History { get; } = new List<ChatMessageRecord>();
-        
-        public ChatSession(T model)
-        {
-            _model = model;
-        }
-
-        public IEnumerable<string> Chat(string text, string? prompt = null, string encoding = "UTF-8")
-        {
-            History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Human, text), DateTime.Now));
-            string totalResponse = "";
-            foreach(var response in _model.Chat(text, prompt, encoding))
-            {
-                totalResponse += response;
-                yield return response;
-            }
-            History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Assistant, totalResponse), DateTime.Now));
-        }
-
-        public ChatSession<T> WithPrompt(string prompt, string encoding = "UTF-8")
-        {
-            _model.InitChatPrompt(prompt, encoding);
-            return this;
-        }
-
-        public ChatSession<T> WithPromptFile(string promptFilename, string encoding = "UTF-8")
-        {
-            return WithPrompt(File.ReadAllText(promptFilename), encoding);
-        }
-
-        /// <summary>
-        /// Set the keyword to split the return value of chat AI.
-        /// </summary>
-        /// <param name="humanName"></param>
-        /// <returns></returns>
-        public ChatSession<T> WithAntiprompt(string[] antiprompt)
-        {
-            _model.InitChatAntiprompt(antiprompt);
-            return this;
-        }
-    }
-}
+    using llama_token = Int32;
+    //public class ChatHistoryEntry
+    //{
+    //    public string Role { get; set; }
+    //    public string Text { get; set; }
+    //}
+
+    //public class ChatMetadata
+    //{
+    //    public string Prompt { get; set; } = "Prompt";
+    //    public IEnumerable<string>? AntiPrompts { get; set; } = null;
+    //    public string User { get; set; } = "User";
+    //    public string Assistant { get; set; } = "Assistant";
+
+    //    public ChatMetadata SetPrompt(string v)
+    //    {
+    //        Prompt = v;
+    //        return this;
+    //    }
+
+    //    public ChatMetadata SetUserName(string v)
+    //    {
+    //        User = v;
+    //        return this;
+    //    }
+
+    //    public ChatMetadata SetAssistantName(string v)
+    //    {
+    //        Assistant = v;
+    //        return this;
+    //    }
+
+    //    public ChatMetadata WithPromptFromFile(string filename)
+    //    {
+    //        Prompt = System.IO.File.ReadAllText(filename);
+    //        return this;
+    //    }
+    //}
+
+    //public class ChatSession
+    //{
+    //    private LLamaModel _model;
+    //    private ChatMetadata _metadata;
+
+    //    public List<ChatHistoryEntry> ChatHistory { get; } = new();
+    //    public ChatSession(LLamaModel model, ChatMetadata? metadata = null)
+    //    {
+    //        _model = model;
+    //        if (metadata == null) metadata = new ChatMetadata();
+    //        _metadata = metadata;
+
+    //        if (_metadata.Prompt != "")
+    //        {
+    //            ChatHistory.Add(new ChatHistoryEntry() { Role = "", Text = _metadata.Prompt });
+    //        }
+    //    }
+
+    //    string _formatChatHistory(List<ChatHistoryEntry> history)
+    //    {
+    //        StringBuilder sb = new();
+    //        foreach (var entry in history)
+    //        {
+    //            if (entry.Role == "")
+    //            {
+    //                sb.Append($"{entry.Text}\n");
+    //                continue;
+    //            }
+    //            sb.Append($"{entry.Role}: {entry.Text}\n");
+    //        }
+    //        sb.Append($"{_metadata.Assistant}: ");
+    //        return sb.ToString();
+    //    }
+
+    //    public IEnumerable<string> Chat(string text)
+    //    {
+    //        ChatHistory.Add(new ChatHistoryEntry() { Role = "User", Text = text });
+    //        string totalResponse = "";
+    //        //foreach (var response in _model.GenerateResult(_formatChatHistory(ChatHistory), null, _metadata.AntiPrompts))
+    //        //{
+    //        //    totalResponse += response;
+    //        //    yield return response;
+    //        //}
+    //        ChatHistory.Add(new ChatHistoryEntry() { Role = "Assistant", Text = totalResponse });
+    //    }
+    //}
+
+    
+
+}

LLama/Common/FixedQuene.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace LLama.Common
+{
+    /// <summary>
+    /// A queue with fixed storage size.
+    /// Currently it's only a naive implementation and needs to be further optimized in the future.
+    /// </summary>
+    public class FixedSizeQuene<T>: IEnumerable<T>
+    {
+        int _maxSize;
+        List<T> _storage;
+
+        public int Count => _storage.Count;
+        public FixedSizeQuene(int size)
+        {
+            _maxSize = size;
+            _storage = new();
+        }
+
+        public FixedSizeQuene<T> FillWith(T value)
+        {
+            for(int i = 0; i < Count; i++)
+            {
+                _storage[i] = value;
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Enquene an element.
+        /// </summary>
+        /// <returns></returns>
+        public void Enqueue(T item)
+        {
+            _storage.Add(item);
+            if(_storage.Count >= _maxSize)
+            {
+                _storage.RemoveAt(0);
+            }
+        }
+
+        public T[] ToArray()
+        {
+            return _storage.ToArray();
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            return _storage.GetEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+}

LLama/ILLamaExecutor.cs

@@ -0,0 +1,12 @@
+using LLama.Abstractions.Params;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama
+{
+    public interface ILLamaExecutor
+    {
+        IEnumerable<string> Infer(string text, SessionParams? sessionParams = null, IEnumerable<string>? antiprompts = null);
+    }
+}

LLama/LLamaExecutorBase.cs

@@ -0,0 +1,111 @@
+using LLama.Abstractions.Params;
+using LLama.Common;
+using LLama.Exceptions;
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace LLama
+{
+    using llama_token = Int32;
+    public abstract class LLamaExecutorBase: ILLamaExecutor
+    {
+        protected LLamaModel _model;
+        protected int _pastTokensCount; // n_past
+        protected int _consumedTokensCount; // n_consume
+        protected int _n_session_consumed;
+        protected int _n_matching_session_tokens;
+        protected string _pathSession;
+        protected List<llama_token> _embeds = new(); // embd
+        protected List<llama_token> _embed_inps = new();
+        protected List<llama_token> _session_tokens = new();
+        protected FixedSizeQuene<llama_token> _last_n_tokens;
+        protected LLamaExecutorBase(LLamaModel model)
+        {
+            _model = model;
+            _pastTokensCount = 0;
+            _consumedTokensCount = 0;
+            _n_session_consumed = 0;
+            _embeds = new();
+            _embed_inps = new();
+            _last_n_tokens = new FixedSizeQuene<llama_token>(_model.ContextSize).FillWith(0);
+        }
+
+        public unsafe LLamaExecutorBase WithSessionFile(string filename)
+        {
+            _pathSession = filename;
+            if (string.IsNullOrEmpty(filename))
+            {
+                throw new ArgumentNullException("File name cannot be empty.");
+            }
+            if (File.Exists(filename))
+            {
+                llama_token[] session_tokens = new llama_token[_model.ContextSize];
+                ulong n_token_count_out = 0;
+                if (!NativeApi.llama_load_session_file(_model.NativeHandle, _pathSession, session_tokens, (ulong)_model.ContextSize, &n_token_count_out))
+                {
+                    throw new RuntimeError($"Failed to load session file {_pathSession}");
+                }
+                _session_tokens = session_tokens.Take((int)n_token_count_out).ToList();
+            }
+            return this;
+        }
+
+        public void SaveSessionFile(string filename)
+        {
+            var session_token_array = _session_tokens.ToArray();
+            NativeApi.llama_save_session_file(_model.NativeHandle, filename, session_token_array, (ulong)session_token_array.Length);
+        }
+
+        protected virtual void HandleRunOutOfContext(int tokensToKeep)
+        {
+            // if we run out of context:
+            // - take the tokensToKeep first tokens from the original prompt (via n_past)
+            // - take half of the last (n_ctx - tokensToKeep) tokens and recompute the logits in batches
+            int n_left = _pastTokensCount - tokensToKeep;
+
+            _pastTokensCount = Math.Max(1, tokensToKeep);
+
+            // insert n_left/2 tokens at the start of embed from last_n_tokens
+            _embeds.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embeds.Count).Skip(_model.ContextSize - n_left / 2 - _embeds.Count));
+
+            // stop saving session if we run out of context
+            _pathSession = string.Empty;
+        }
+
+        protected virtual void TryReuseMathingPrefix()
+        {
+            if (_n_session_consumed < _session_tokens.Count)
+            {
+                int i = 0;
+                for (; i < _embeds.Count; i++)
+                {
+                    if (_embeds[i] != _session_tokens[_n_session_consumed])
+                    {
+                        _session_tokens = _session_tokens.Take(_n_session_consumed).ToList();
+                        break;
+                    }
+
+                    _pastTokensCount++;
+                    _n_session_consumed++;
+
+                    if (_n_session_consumed >= _session_tokens.Count)
+                    {
+                        i++;
+                        break;
+                    }
+                }
+
+                if (i > 0)
+                {
+                    _embeds.RemoveRange(0, i);
+                }
+            }
+        }
+
+        public abstract IEnumerable<string> Infer(string text, SessionParams? sessionParams = null, IEnumerable<string>? antiprompts = null);
+    }
+}

LLama/LLamaInstructExecutor.cs

@@ -0,0 +1,200 @@
+using LLama.Abstractions.Params;
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace LLama
+{
+    using llama_token = Int32;
+    public class LLamaInstructExecutor : LLamaExecutorBase
+    {
+        bool _prompt_run = true;
+        readonly IEnumerable<llama_token> _llama_token_newline;
+        readonly IEnumerable<llama_token> _inp_pfx;
+        readonly IEnumerable<llama_token> _inp_sfx;
+        public LLamaInstructExecutor(LLamaModel model, string inputPrefix = "\n\n### Instruction:\n\n",
+            string inputSuffix = "\n\n### Response:\n\n") : base(model)
+        {
+            _llama_token_newline = Utils.Tokenize(_model.NativeHandle, "\n", false, _model.Encoding);
+            _inp_pfx = _model.Tokenize(inputPrefix, true);
+            _inp_sfx = _model.Tokenize(inputSuffix, false);
+        }
+
+        /// <summary>
+        /// process the text and return the tokens consumed.
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="sessionParams"></param>
+        /// <param name="encoding"></param>
+        /// <param name="is_antiprompt"></param>
+        /// <returns></returns>
+        protected virtual int ProcessTextBeforeInfer(string text, SessionParams sessionParams)
+        {
+            if (text.Length > 1)
+            {
+                if (!text.EndsWith("\n"))
+                {
+                    text += "\n";
+                }
+                _consumedTokensCount = _embed_inps.Count;
+                _embed_inps.AddRange(_inp_pfx);
+
+                var line_inp = _model.Tokenize(text, false);
+                _embed_inps.AddRange(line_inp);
+
+                _embed_inps.AddRange(_inp_sfx);
+
+                return line_inp.Count();
+            }
+            else
+            {
+                return 0;
+            }
+        }
+
+        public override IEnumerable<string> Infer(string text, SessionParams? sessionParams = null, IEnumerable<string>? antiprompts = null)
+        {
+            if (sessionParams is null)
+            {
+                sessionParams = new SessionParams();
+            }
+            // if n_remain < 0, the response will be generated endlessly.
+            int n_remain = sessionParams.ResponseTokensCount;
+            bool return_value = false;
+            bool wait_for_input = false;
+            bool need_to_save_session = !string.IsNullOrEmpty(_pathSession) && _n_matching_session_tokens < _embed_inps.Count;
+
+            if (_prompt_run)
+            {
+                // When running the first input (prompt) in inteactive mode, we should specially process it.
+                text = " " + text;
+                _embed_inps = _model.Tokenize(text, true).ToList();
+            }
+            else
+            {
+                n_remain -= ProcessTextBeforeInfer(text, sessionParams);
+            }
+
+            while (n_remain != 0 || _prompt_run)
+            {
+                if (_embeds.Count > 0)
+                {
+                    _prompt_run = false;
+                    if (_pastTokensCount + _embeds.Count > _model.ContextSize)
+                    {
+                        HandleRunOutOfContext(sessionParams.TokensToKeep);
+                    }
+
+                    TryReuseMathingPrefix();
+                    _pastTokensCount = _model.Eval(_embeds.ToArray(), _pastTokensCount);
+
+                    if (_embeds.Count > 0 && !string.IsNullOrEmpty(_pathSession))
+                    {
+                        _session_tokens.AddRange(_embeds);
+                        _n_session_consumed = _session_tokens.Count;
+                    }
+                }
+
+                _embeds.Clear();
+
+                if (_embed_inps.Count <= _consumedTokensCount && !wait_for_input)
+                {
+                    var temp = sessionParams.Temperature;
+                    var top_k = sessionParams.TopK <= 0 ? NativeApi.llama_n_vocab(_model.NativeHandle) : sessionParams.TopK;
+                    var top_p = sessionParams.TopK;
+                    var tfs_z = sessionParams.TfsZ;
+                    var typical_p = sessionParams.TypicalP;
+                    var repeat_last_n = sessionParams.RepeatLastTokensCount < 0 ? _model.ContextSize : sessionParams.RepeatLastTokensCount;
+                    var repeat_penalty = sessionParams.RepeatPenalty;
+                    var alpha_presence = sessionParams.PresencePenalty;
+                    var alpha_frequency = sessionParams.FrequencyPenalty;
+                    var mirostat = sessionParams.Mirostat;
+                    var mirostat_tau = sessionParams.MirostatTau;
+                    var mirostat_eta = sessionParams.MirostatEta;
+                    var penalize_nl = sessionParams.PenalizeNL;
+
+                    // optionally save the session on first sample (for faster prompt loading next time)
+                    if (!string.IsNullOrEmpty(_pathSession) && need_to_save_session)
+                    {
+                        need_to_save_session = false;
+                        SaveSessionFile(_pathSession);
+                    }
+
+                    var tokenDataArray = _model.ApplyPenalty(_last_n_tokens, sessionParams.LogitBias, repeat_last_n,
+                        repeat_penalty, alpha_frequency, alpha_presence, penalize_nl);
+
+                    var id = _model.Sample(tokenDataArray, temp, mirostat, mirostat_tau, mirostat_eta, top_k, top_p,
+                        tfs_z, typical_p);
+
+                    _last_n_tokens.Enqueue(id);
+
+                    _embeds.Add(id);
+
+                    n_remain--;
+                    return_value = true;
+                }
+                else
+                {
+                    while (_embed_inps.Count > _consumedTokensCount)
+                    {
+                        _embeds.Add(_embed_inps[_consumedTokensCount]);
+                        _last_n_tokens.Enqueue(_embed_inps[_consumedTokensCount]);
+                        _consumedTokensCount++;
+                        if (_embeds.Count >= _model.Params.BatchSize)
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                if (return_value)
+                {
+                    foreach (var item in _model.GenerateResult(_embeds))
+                    {
+                        yield return item;
+                    }
+                }
+
+                if (_embed_inps.Count <= _consumedTokensCount)
+                {
+                    if (antiprompts is not null && antiprompts.Count() > 0)
+                    {
+                        string last_output = "";
+                        foreach (var id in _last_n_tokens)
+                        {
+                            last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding);
+                        }
+
+                        foreach (var antiprompt in antiprompts)
+                        {
+                            if (last_output.EndsWith(antiprompt))
+                            {
+                                wait_for_input = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (_pastTokensCount > 0 && wait_for_input)
+                    {
+                        yield return "\n> ";
+                        break;
+                    }
+                }
+
+                if (_embeds.Count > 0 && _embeds.Last() == NativeApi.llama_token_eos())
+                {
+                    wait_for_input = true;
+                }
+
+                if (n_remain <= 0 && sessionParams.ResponseTokensCount != -1)
+                {
+                    n_remain = sessionParams.ResponseTokensCount;
+                    wait_for_input = true;
+                }
+            }
+        }
+    }
+}

LLama/LLamaInteractExecutor.cs

@@ -0,0 +1,203 @@
+using LLama.Abstractions.Params;
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace LLama
+{
+    using llama_token = Int32;
+    public class LLamaInteractExecutor : LLamaExecutorBase
+    {
+        bool _prompt_run = true;
+        readonly IEnumerable<llama_token> _llama_token_newline;
+        readonly IEnumerable<llama_token> _inp_pfx;
+        readonly IEnumerable<llama_token> _inp_sfx;
+        public LLamaInteractExecutor(LLamaModel model) : base(model)
+        {
+            _llama_token_newline = Utils.Tokenize(_model.NativeHandle, "\n", false, _model.Encoding);
+            _inp_pfx = _model.Tokenize("\n\n### Instruction:\n\n", true);
+            _inp_sfx = _model.Tokenize("\n\n### Response:\n\n", false);
+        }
+
+        /// <summary>
+        /// process the text and return the tokens consumed.
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="sessionParams"></param>
+        /// <param name="encoding"></param>
+        /// <param name="is_antiprompt"></param>
+        /// <returns></returns>
+        protected virtual int ProcessTextBeforeInfer(string text, SessionParams sessionParams)
+        {
+            if (text.Length > 1)
+            {
+                if (!text.EndsWith("\n"))
+                {
+                    text += "\n";
+                }
+                var line_inp = _model.Tokenize(text, false);
+                _embed_inps.AddRange(line_inp);
+                return line_inp.Count();
+            }
+            else
+            {
+                return 0;
+            }
+        }
+
+        public override IEnumerable<string> Infer(string text, SessionParams? sessionParams = null, IEnumerable<string>? antiprompts = null)
+        {
+            if (sessionParams is null)
+            {
+                sessionParams = new SessionParams();
+            }
+            // if n_remain < 0, the response will be generated endlessly.
+            int n_remain = sessionParams.ResponseTokensCount;
+            bool return_value = false;
+            bool wait_for_input = false;
+            bool need_to_save_session = !string.IsNullOrEmpty(_pathSession) && _n_matching_session_tokens < _embed_inps.Count;
+
+            if (_prompt_run)
+            {
+                // When running the first input (prompt) in inteactive mode, we should specially process it.
+                text = " " + text;
+                _embed_inps = _model.Tokenize(text, true).ToList();
+            }
+            else
+            {
+                n_remain -= ProcessTextBeforeInfer(text, sessionParams);
+            }
+
+            while (n_remain != 0 && !wait_for_input || _prompt_run)
+            {
+                if (_embeds.Count > 0)
+                {
+                    _prompt_run = false;
+                    if (_pastTokensCount + _embeds.Count > _model.ContextSize)
+                    {
+                        HandleRunOutOfContext(sessionParams.TokensToKeep);
+                    }
+
+                    TryReuseMathingPrefix();
+                    _pastTokensCount = _model.Eval(_embeds.ToArray(), _pastTokensCount);
+
+                    if (_embeds.Count > 0 && !string.IsNullOrEmpty(_pathSession))
+                    {
+                        _session_tokens.AddRange(_embeds);
+                        _n_session_consumed = _session_tokens.Count;
+                    }
+                }
+
+                _embeds.Clear();
+
+                if (_embed_inps.Count <= _consumedTokensCount && !wait_for_input)
+                {
+                    var temp = sessionParams.Temperature;
+                    var top_k = sessionParams.TopK <= 0 ? NativeApi.llama_n_vocab(_model.NativeHandle) : sessionParams.TopK;
+                    var top_p = sessionParams.TopK;
+                    var tfs_z = sessionParams.TfsZ;
+                    var typical_p = sessionParams.TypicalP;
+                    var repeat_last_n = sessionParams.RepeatLastTokensCount < 0 ? _model.ContextSize : sessionParams.RepeatLastTokensCount;
+                    var repeat_penalty = sessionParams.RepeatPenalty;
+                    var alpha_presence = sessionParams.PresencePenalty;
+                    var alpha_frequency = sessionParams.FrequencyPenalty;
+                    var mirostat = sessionParams.Mirostat;
+                    var mirostat_tau = sessionParams.MirostatTau;
+                    var mirostat_eta = sessionParams.MirostatEta;
+                    var penalize_nl = sessionParams.PenalizeNL;
+
+                    // optionally save the session on first sample (for faster prompt loading next time)
+                    if (!string.IsNullOrEmpty(_pathSession) && need_to_save_session)
+                    {
+                        need_to_save_session = false;
+                        SaveSessionFile(_pathSession);
+                    }
+
+                    var tokenDataArray = _model.ApplyPenalty(_last_n_tokens, sessionParams.LogitBias, repeat_last_n,
+                        repeat_penalty, alpha_frequency, alpha_presence, penalize_nl);
+
+                    var id = _model.Sample(tokenDataArray, temp, mirostat, mirostat_tau, mirostat_eta, top_k, top_p,
+                        tfs_z, typical_p);
+
+                    _last_n_tokens.Enqueue(id);
+
+                    if (id == NativeApi.llama_token_eos())
+                    {
+                        id = _llama_token_newline.First();
+                        if (antiprompts is not null && antiprompts.Count() > 0)
+                        {
+                            var first_antiprompt = _model.Tokenize(antiprompts.First(), false);
+                            _embed_inps.AddRange(first_antiprompt);
+                        }
+                    }
+
+                    _embeds.Add(id);
+
+                    n_remain--;
+                    return_value = true;
+                }
+                else
+                {
+                    while (_embed_inps.Count > _consumedTokensCount)
+                    {
+                        _embeds.Add(_embed_inps[_consumedTokensCount]);
+                        _last_n_tokens.Enqueue(_embed_inps[_consumedTokensCount]);
+                        _consumedTokensCount++;
+                        if (_embeds.Count >= _model.Params.BatchSize)
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                if (return_value)
+                {
+                    foreach (var item in _model.GenerateResult(_embeds))
+                    {
+                        yield return item;
+                    }
+                }
+
+                if (_embed_inps.Count <= _consumedTokensCount)
+                {
+                    if (antiprompts is not null && antiprompts.Count() > 0)
+                    {
+                        string last_output = "";
+                        foreach (var id in _last_n_tokens)
+                        {
+                            last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding);
+                        }
+
+                        foreach (var antiprompt in antiprompts)
+                        {
+                            if (last_output.EndsWith(antiprompt))
+                            {
+                                wait_for_input = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (_pastTokensCount > 0 && wait_for_input)
+                    {
+                        break;
+                    }
+                }
+
+                if (_embeds.Count > 0 && _embeds.Last() == NativeApi.llama_token_eos())
+                {
+                    yield return " [end of text]\n";
+                    break;
+                }
+
+                if (n_remain <= 0 && sessionParams.ResponseTokensCount != -1)
+                {
+                    n_remain = sessionParams.ResponseTokensCount;
+                    wait_for_input = true;
+                }
+            }
+        }
+    }
+}

LLama/LLamaModel.cs

@@ -1,804 +1,188 @@
-using LLama.Exceptions;
+using LLama.Abstractions.Params;
+using LLama.Exceptions;
+using LLama.Native;
+using LLama.Old;
 using LLama.Types;
 using LLama.Extensions;
-using LLama.Native;
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 
 namespace LLama
 {
     using llama_token = Int32;
-    public class LLamaModel : IChatModel, IDisposable
+    public class LLamaModel
     {
-        LLamaParams _params;
+        // TODO: expose more properties.
+        LLamaLogger _logger;
+        Encoding _encoding;
         SafeLLamaContextHandle _ctx;
-        string _path_session;
-        List<llama_token> _session_tokens;
-        List<llama_token> _embed_inp;
-        int _n_ctx;
-        List<llama_token> _inp_pfx;
-        List<llama_token> _inp_sfx;
-        List<llama_token> _llama_token_newline;
-        List<llama_token> _last_n_tokens;
-        bool _is_interacting;
-        bool _is_antiprompt;
-        bool _input_echo;
-        bool _verbose;
-
-        // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session
-        // if we loaded a session with at least 75% similarity. It's currently just used to speed up the
-        // initial prompt so it doesn't need to be an exact match.
-        bool _need_to_save_session;
-        int _n_past;
-        int _n_remain;
-        int _n_consumed;
-        int _n_session_consumed;
-        List<llama_token> _embed;
-
-        public string Name { get; set; }
-        public bool Verbose
-        {
-            get
-            {
-                return _verbose;
-            }
-            set
-            {
-                _verbose = value;
-            }
-        }
+        public int ContextSize { get; }
+        public ModelParams Params { get; set; }
         public SafeLLamaContextHandle NativeHandle => _ctx;
+        public Encoding Encoding => _encoding;
 
-        /// <summary>
-        /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will 
-        /// load 20 layers to gpu by default.
-        /// </summary>
-        /// <param name="model_path">The model file path.</param>
-        /// <param name="model_name">The model name.</param>
-        /// <param name="verbose">Whether to print details when running the model.</param>
-        /// <param name="seed"></param>
-        /// <param name="n_threads"></param>
-        /// <param name="n_predict"></param>
-        /// <param name="n_ctx"></param>
-        /// <param name="n_batch"></param>
-        /// <param name="n_keep"></param>
-        /// <param name="n_gpu_layers"></param>
-        /// <param name="logit_bias"></param>
-        /// <param name="top_k"></param>
-        /// <param name="top_p"></param>
-        /// <param name="tfs_z"></param>
-        /// <param name="typical_p"></param>
-        /// <param name="temp"></param>
-        /// <param name="repeat_penalty"></param>
-        /// <param name="repeat_last_n"></param>
-        /// <param name="frequency_penalty"></param>
-        /// <param name="presence_penalty"></param>
-        /// <param name="mirostat"></param>
-        /// <param name="mirostat_tau"></param>
-        /// <param name="mirostat_eta"></param>
-        /// <param name="prompt"></param>
-        /// <param name="path_session"></param>
-        /// <param name="input_prefix"></param>
-        /// <param name="input_suffix"></param>
-        /// <param name="antiprompt"></param>
-        /// <param name="lora_adapter"></param>
-        /// <param name="lora_base"></param>
-        /// <param name="memory_f16"></param>
-        /// <param name="random_prompt"></param>
-        /// <param name="use_color"></param>
-        /// <param name="interactive"></param>
-        /// <param name="embedding"></param>
-        /// <param name="interactive_first"></param>
-        /// <param name="prompt_cache_all"></param>
-        /// <param name="instruct"></param>
-        /// <param name="penalize_nl"></param>
-        /// <param name="perplexity"></param>
-        /// <param name="use_mmap"></param>
-        /// <param name="use_mlock"></param>
-        /// <param name="mem_test"></param>
-        /// <param name="verbose_prompt"></param>
-        /// <param name="encoding"></param>
-        public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1,
-            int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1,
-            Dictionary<llama_token, float> logit_bias = null, int top_k = 40, float top_p = 0.95f,
-            float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f,
-            int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f,
-            int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "",
-            string path_session = "", string input_prefix = "", string input_suffix = "",
-            List<string> antiprompt = null, string lora_adapter = "", string lora_base = "",
-            bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false,
-            bool embedding = false, bool interactive_first = false, bool prompt_cache_all = false, bool instruct = false, bool penalize_nl = true,
-            bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false,
-            bool verbose_prompt = false, string encoding = "UTF-8") : this(new LLamaParams(seed: seed,
-                n_threads: n_threads,
-                n_predict: n_predict,
-                n_ctx: n_ctx,
-                n_batch: n_batch,
-                n_keep: n_keep,
-                n_gpu_layers: n_gpu_layers,
-                logit_bias: logit_bias,
-                top_k: top_k,
-                top_p: top_p,
-                tfs_z: tfs_z,
-                typical_p: typical_p,
-                temp: temp,
-                repeat_penalty: repeat_penalty,
-                repeat_last_n: repeat_last_n,
-                frequency_penalty: frequency_penalty,
-                presence_penalty: presence_penalty,
-                mirostat: mirostat,
-                mirostat_tau: mirostat_tau,
-                mirostat_eta: mirostat_eta,
-                model: model_path,
-                prompt: prompt,
-                path_session: path_session,
-                input_prefix: input_prefix,
-                input_suffix: input_suffix,
-                antiprompt: antiprompt,
-                lora_adapter: lora_adapter,
-                lora_base: lora_base,
-                memory_f16: memory_f16,
-                random_prompt: random_prompt,
-                use_color: use_color,
-                interactive: interactive,
-                embedding: embedding,
-                interactive_first: interactive_first,
-                prompt_cache_all: prompt_cache_all,
-                instruct: instruct,
-                penalize_nl: penalize_nl,
-                perplexity: perplexity,
-                use_mmap: use_mmap,
-                use_mlock: use_mlock,
-                mem_test: mem_test,
-                verbose_prompt: verbose_prompt),
-                model_name, verbose, encoding)
+        public void Dispose()
         {
-
+            _ctx.Dispose();
         }
 
-        /// <summary>
-        /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will 
-        /// load 20 layers to gpu by default.
-        /// </summary>
-        /// <param name="params">The LLamaModel params</param>
-        /// <param name="name">Model name</param>
-        /// <param name="verbose">Whether to output the detailed info.</param>
-        /// <param name="encoding"></param>
-        /// <exception cref="RuntimeError"></exception>
-        public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8")
+        public LLamaModel(ModelParams Params, string encoding = "UTF-8")
         {
-            Name = name;
-            _params = @params;
-            _verbose = verbose;
-            _ctx = Utils.llama_init_from_gpt_params(ref _params);
-
-            // Add a space in front of the first character to match OG llama tokenizer behavior
-            _session_tokens = new List<llama_token>();
-
-            _path_session = @params.path_session;
-            if (!string.IsNullOrEmpty(_path_session))
-            {
-                if (verbose)
-                {
-                    LLamaLogger.Default.Info($"Attempting to load saved session from '{_path_session}'");
-                }
-
-                if (!File.Exists(_path_session))
-                {
-                    LLamaLogger.Default.Warn("Session file does not exist, will create.");
-                }
-
-                llama_token[] session_tokens = new llama_token[@params.n_ctx];
-                ulong n_token_count_out = 0;
-                if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out))
-                {
-                    throw new RuntimeError($"Failed to load session file {_path_session}");
-                }
-                _session_tokens = session_tokens.Take((int)n_token_count_out).ToList();
-                if (verbose)
-                {
-                    LLamaLogger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens");
-                }
-            }
-
-            _n_ctx = NativeApi.llama_n_ctx(_ctx);
-
-            WithPrompt(_params.prompt);
-
-            // prefix & suffix for instruct mode
-            _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true, encoding);
-            _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false, encoding);
-
-            // in instruct mode, we inject a prefix and a suffix to each input by the user
-            if (_params.instruct)
-            {
-                _params.interactive_first = true;
-                _params.antiprompt.Add("### Instruction:\n\n");
-            }
-
-            // enable interactive mode if reverse prompt or interactive start is specified
-            if (_params.interactive_first)
-            {
-                _params.interactive = true;
-            }
-
-            // determine newline token
-            _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false, encoding);
-
-            if (_params.verbose_prompt)
-            {
-                LLamaLogger.Default.Info("\n");
-                LLamaLogger.Default.Info($"prompt: '{_params.prompt}'");
-                LLamaLogger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}");
-                for (int i = 0; i < _embed_inp.Count; i++)
-                {
-                    LLamaLogger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'");
-                }
-                if (_params.n_keep > 0)
-                {
-                    LLamaLogger.Default.Info($"static prompt based on n_keep: '");
-                    for (int i = 0; i < _params.n_keep; i++)
-                    {
-                        LLamaLogger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}");
-                    }
-                    LLamaLogger.Default.Info("\n");
-                }
-                LLamaLogger.Default.Info("\n");
-            }
-
-            if (_params.interactive && verbose)
-            {
-                LLamaLogger.Default.Info("interactive mode on.");
-            }
-            if (verbose)
-            {
-                LLamaLogger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " +
-                    $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " +
-                    $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," +
-                    $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," +
-                    $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}");
-                LLamaLogger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " +
-                    $"n_keep = {_params.n_keep}");
-                LLamaLogger.Default.Info("\n");
-            }
-
-            _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList();
-
-            if (_params.interactive)
-            {
-                if (verbose)
-                {
-                    LLamaLogger.Default.Info("== Running in interactive mode. ==");
-                }
-                _is_interacting = _params.interactive_first;
-            }
-
-            _is_antiprompt = false;
-            _input_echo = false;
-            _n_past = 0;
-            _n_remain = _params.n_predict;
-            _n_consumed = 0;
-            _n_session_consumed = 0;
-            _embed = new List<llama_token>();
+            _logger = LLamaLogger.Default;
+            this.Params = Params;
+            _encoding = Encoding.GetEncoding(encoding);
+            _logger.Info($"Initializing LLama model with params: {this.Params}");
+            _ctx = Utils.InitLLamaContextFromModelParams(this.Params);
+            ContextSize = NativeApi.llama_n_ctx(_ctx);
         }
 
         /// <summary>
-        /// Apply a prompt to the model.
+        /// Tokenize a string.
         /// </summary>
-        /// <param name="prompt"></param>
-        /// <param name="encoding"></param>
+        /// <param name="text"></param>
+        /// <param name="addBos">Whether to add a bos to the text.</param>
         /// <returns></returns>
-        /// <exception cref="ArgumentException"></exception>
-        public LLamaModel WithPrompt(string prompt, string encoding = "UTF-8")
+        public IEnumerable<llama_token> Tokenize(string text, bool addBos = true)
         {
-            _params.prompt = prompt.Insert(0, " ");
-            _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true, encoding);
-
-            if (_embed_inp.Count > _n_ctx - 4)
-            {
-                throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})");
-            }
-
-            ulong n_matching_session_tokens = 0;
-            if (_session_tokens.Count > 0)
-            {
-                foreach (var id in _session_tokens)
-                {
-                    if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens])
-                    {
-                        break;
-                    }
-                    n_matching_session_tokens++;
-                }
-                if (n_matching_session_tokens >= (ulong)_embed_inp.Count)
-                {
-                    LLamaLogger.Default.Info("Session file has exact match for prompt!");
-                }
-                else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2))
-                {
-                    LLamaLogger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " +
-                        $"/ {_embed_inp.Count} tokens); will mostly be reevaluated.");
-                }
-                else
-                {
-                    LLamaLogger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " +
-                        $"tokens of prompt.");
-                }
-            }
-            // number of tokens to keep when resetting context
-            if (_params.n_keep < 0 || _params.n_keep > (int)_embed_inp.Count || _params.instruct)
-            {
-                _params.n_keep = _embed_inp.Count;
-            }
-            if (_embed_inp.Count > _n_ctx - 4)
-            {
-                throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})");
-            }
-            _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4);
-            
-            return this;
+            // TODO: reconsider whether to convert to array here.
+            return Utils.Tokenize(_ctx, text, addBos, _encoding);
         }
 
         /// <summary>
-        /// Apply the prompt file to the model.
+        /// Detokenize the tokens to text.
         /// </summary>
-        /// <param name="promptFileName"></param>
+        /// <param name="tokens"></param>
         /// <returns></returns>
-        public LLamaModel WithPromptFile(string promptFileName)
+        public string DeTokenize(IEnumerable<llama_token> tokens)
         {
-            return WithPrompt(File.ReadAllText(promptFileName));
+            StringBuilder sb = new();
+            foreach(var token in tokens)
+            {
+                sb.Append(Utils.PtrToString(NativeApi.llama_token_to_str(_ctx, token), _encoding));
+            }
+            return sb.ToString();
         }
 
-        private void ProcessTextBeforeInfer(string text, string encoding)
+        public llama_token Sample(LLamaTokenDataArray candidates, float temperature = 0.8f, MiroStateType mirostat = MiroStateType.Disable, 
+            float mirostatTau = 5.0f, float mirostatEta = 0.1f, int topK = 40, float topP = 0.95f, float tfsZ = 1.0f, float typicalP = 1.0f)
         {
-            if (!string.IsNullOrEmpty(_params.input_prefix))
+            llama_token id = 0;
+            if (temperature <= 0)
             {
-                text = _params.input_prefix + text;
+                // Greedy sampling
+                id = SamplingApi.llama_sample_token_greedy(_ctx, candidates);
             }
-            //if (!text.EndsWith("\n"))
-            //{
-            //    text += "\n";
-            //}
-            if (text.Length > 1)
+            else
             {
-                // append input suffix if any
-                if (!string.IsNullOrEmpty(_params.input_suffix))
+                if (mirostat == MiroStateType.MiroState)
                 {
-                    text += _params.input_suffix;
-                    //yield return _params.input_suffix;
+                    float mirostat_mu = 2.0f * mirostatTau;
+                    const int mirostat_m = 100;
+                    SamplingApi.llama_sample_temperature(_ctx, candidates, temperature);
+                    id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates, mirostatTau, mirostatEta, mirostat_m, ref mirostat_mu);
                 }
-
-                // instruct mode: insert instruction prefix
-                if (_params.instruct && !_is_antiprompt)
+                else if (mirostat == MiroStateType.MiroState2)
                 {
-                    _n_consumed = _embed_inp.Count;
-                    _embed_inp.AddRange(_inp_pfx);
+                    float mirostat_mu = 2.0f * mirostatTau;
+                    SamplingApi.llama_sample_temperature(_ctx, candidates, temperature);
+                    id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates, mirostatTau, mirostatEta, ref mirostat_mu);
                 }
-
-                var line_inp = Utils.llama_tokenize(_ctx, text, false, encoding);
-                _embed_inp.AddRange(line_inp);
-
-                // instruct mode: insert response suffix
-                if (_params.instruct)
+                else
                 {
-                    _embed_inp.AddRange(_inp_sfx);
+                    // Temperature sampling
+                    SamplingApi.llama_sample_top_k(_ctx, candidates, topK, 1);
+                    SamplingApi.llama_sample_tail_free(_ctx, candidates, tfsZ, 1);
+                    SamplingApi.llama_sample_typical(_ctx, candidates, typicalP, 1);
+                    SamplingApi.llama_sample_top_p(_ctx, candidates, topP, 1);
+                    SamplingApi.llama_sample_temperature(_ctx, candidates, temperature);
+                    id = SamplingApi.llama_sample_token(_ctx, candidates);
                 }
-
-                _n_remain -= line_inp.Count;
             }
+            return id;
         }
 
-        public void InitChatPrompt(string prompt, string encoding = "UTF-8")
+        public LLamaTokenDataArray ApplyPenalty(IEnumerable<llama_token> lastTokens, Dictionary<llama_token, float>? logitBias = null, 
+            int repeatLastTokensCount = 64, float repeatPenalty = 1.1f, float alphaFrequency = .0f, float alphaPresence = .0f, 
+            bool penalizeNL = true)
         {
-            WithPrompt(prompt);
-        }
+            var n_vocab = NativeApi.llama_n_vocab(_ctx);
+            var logits = Utils.GetLogits(_ctx, n_vocab);
 
-        public void InitChatAntiprompt(string[] antiprompt)
-        {
-            _params.antiprompt = antiprompt.ToList();
-        }
-
-        /// <summary>
-        /// Chat with the LLaMa model under interactive mode.
-        /// </summary>
-        /// <param name="text"></param>
-        /// <param name="prompt"></param>
-        /// <param name="encoding"></param>
-        /// <returns></returns>
-        /// <exception cref="ArgumentException"></exception>
-        public IEnumerable<string> Chat(string text, string? prompt = null, string encoding = "UTF-8")
-        {
-            if (!_params.interactive)
-            {
-                throw new ArgumentException("The chat API could be only used under interactive model.");
-            }
-            _input_echo = false;
-            if (!string.IsNullOrEmpty(prompt))
+            // Apply params.logit_bias map
+            if(logitBias is not null)
             {
-                WithPrompt(prompt);
+                foreach (var (key, value) in logitBias)
+                {
+                    logits[key] += value;
+                }
             }
-            return Call(text, encoding);
-        }
-
-        /// <summary>
-        /// Save the state to specified path.
-        /// </summary>
-        /// <param name="filename"></param>
-        public void SaveState(string filename)
-        {
-            var stateSize = NativeApi.llama_get_state_size(_ctx);
-            byte[] stateMemory = new byte[stateSize];
-            NativeApi.llama_copy_state_data(_ctx, stateMemory);
-            File.WriteAllBytes(filename, stateMemory);
-        }
 
-        /// <summary>
-        /// Load the state from specified path.
-        /// </summary>
-        /// <param name="filename"></param>
-        /// <param name="clearPreviousEmbed">Whether to clear previous footprints of this model.</param>
-        /// <exception cref="RuntimeError"></exception>
-        public void LoadState(string filename, bool clearPreviousEmbed = true)
-        {
-            var stateMemory = File.ReadAllBytes(filename);
-            int stateSize = (int)NativeApi.llama_get_state_size(_ctx);
-            if (stateMemory.Length != stateSize)
+            var candidates = new List<LLamaTokenData>();
+            candidates.Capacity = n_vocab;
+            for (llama_token token_id = 0; token_id < n_vocab; token_id++)
             {
-                throw new RuntimeError("Failed to validate state size.");
+                candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f));
             }
-            NativeApi.llama_set_state_data(_ctx, stateMemory);
 
-            if (clearPreviousEmbed)
-            {
-                WithPrompt(_params.prompt);
-            }
-        }
+            LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false);
 
-        /// <summary>
-        /// Tokenize a string.
-        /// </summary>
-        /// <param name="text">The utf-8 encoded string to tokenize.</param>
-        /// <returns>A list of tokens.</returns>
-        /// <exception cref="RuntimeError">If the tokenization failed.</exception>
-        public List<llama_token> Tokenize(string text, string encoding = "UTF-8")
-        {
-            Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero);
-            var n_ctx = NativeApi.llama_n_ctx(_ctx);
-            var tokens = new llama_token[n_ctx];
-            var n_tokens = NativeApi.llama_tokenize(_ctx, text, Encoding.GetEncoding(encoding), tokens, n_ctx, true);
-            if (n_tokens < 0)
+            // Apply penalties
+            float nl_logit = logits[NativeApi.llama_token_nl()];
+            int lastTokensCount = lastTokens.Count();
+            var last_n_repeat = Math.Min(Math.Min(lastTokensCount, repeatLastTokensCount), ContextSize);
+            SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
+                lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
+                (ulong)last_n_repeat, repeatPenalty);
+            SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
+                lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
+                (ulong)last_n_repeat, alphaFrequency, alphaPresence);
+            if (!penalizeNL)
             {
-                throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}");
+                logits[NativeApi.llama_token_nl()] = nl_logit;
             }
-            return tokens.Take(n_tokens).ToList();
-        }
 
-        /// <summary>
-        /// Detokenize a list of tokens.
-        /// </summary>
-        /// <param name="tokens">The list of tokens to detokenize.</param>
-        /// <returns>The detokenized string.</returns>
-        public string DeTokenize(IEnumerable<llama_token> tokens)
-        {
-            Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero);
-            string output = "";
-            foreach (var token in tokens)
-            {
-                output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token));
-            }
-            return output;
+            return candidates_p;
         }
 
         /// <summary>
-        /// Call the model to run inference.
+        /// 
         /// </summary>
-        /// <param name="text"></param>
-        /// <param name="encoding"></param>
-        /// <returns></returns>
+        /// <param name="tokens"></param>
+        /// <param name="pastTokensCount"></param>
+        /// <returns>The updated `pastTokensCount`.</returns>
         /// <exception cref="RuntimeError"></exception>
-        public IEnumerable<string> Call(string text, string encoding = "UTF-8")
+        public llama_token Eval(llama_token[] tokens, llama_token pastTokensCount)
         {
-            _is_antiprompt = false;
-            if(_n_past > 0)
-            {
-                _is_interacting = false;
-            }
-            if (_is_interacting)
-            {
-                if (_verbose)
-                {
-                    LLamaLogger.Default.Warn("In interacting when calling the model, automatically changed it.");
-                }
-                _is_interacting = false;
-            }
-            ProcessTextBeforeInfer(text, encoding);
-
-            while ((_n_remain != 0 || _params.interactive) && !_is_interacting)
+            int total = tokens.Length;
+            for(int i = 0; i < total; i += Params.BatchSize)
             {
-                if (_embed.Count > 0)
+                int n_eval = total - i;
+                if(n_eval > Params.BatchSize)
                 {
-                    // infinite text generation via context swapping
-                    // if we run out of context:
-                    // - take the n_keep first tokens from the original prompt (via n_past)
-                    // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches
-                    if (_n_past + _embed.Count > _n_ctx)
-                    {
-                        int n_left = _n_past - _params.n_keep;
-
-                        _n_past = Math.Max(1, _params.n_keep);
-
-                        // insert n_left/2 tokens at the start of embed from last_n_tokens
-                        _embed.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embed.Count).Skip(_n_ctx - n_left / 2 - _embed.Count));
-
-                        // stop saving session if we run out of context
-                        _path_session = "";
-                    }
-
-                    // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
-                    // REVIEW
-                    if (_n_session_consumed < _session_tokens.Count)
-                    {
-                        int i = 0;
-                        for (; i < _embed.Count; i++)
-                        {
-                            if (_embed[i] != _session_tokens[_n_session_consumed])
-                            {
-                                _session_tokens = _session_tokens.Take(_n_session_consumed).ToList();
-                                break;
-                            }
-
-                            _n_past++;
-                            _n_session_consumed++;
-
-                            if (_n_session_consumed >= _session_tokens.Count)
-                            {
-                                i++;
-                                break;
-                            }
-                        }
-
-                        if (i > 0)
-                        {
-                            _embed.RemoveRange(0, i);
-                        }
-                    }
-
-                    // evaluate tokens in batches
-                    // embed is typically prepared beforehand to fit within a batch, but not always
-                    for (int i = 0; i < _embed.Count; i += _params.n_batch)
-                    {
-                        int n_eval = _embed.Count - i;
-
-                        if (n_eval > _params.n_batch)
-                        {
-                            n_eval = _params.n_batch;
-                        }
-
-                        var array = _embed.Skip(i).ToArray();
-                        if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0)
-                        {
-                            LLamaLogger.Default.Error($"Failed to eval.");
-                            throw new RuntimeError("Failed to eval.");
-                        }
-
-                        _n_past += n_eval;
-                    }
-
-                    if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session))
-                    {
-                        _session_tokens.AddRange(_embed);
-                        _n_session_consumed = _session_tokens.Count;
-                    }
+                    n_eval = Params.BatchSize;
                 }
 
-                _embed.Clear();
-
-                if (_embed_inp.Count <= _n_consumed && !_is_interacting)
+                if(Utils.Eval(_ctx, tokens, i, n_eval, pastTokensCount, Params.Threads) != 0)
                 {
-                    var temp = _params.temp;
-                    var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k;
-                    var top_p = _params.top_p;
-                    var tfs_z = _params.tfs_z;
-                    var typical_p = _params.typical_p;
-                    var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n;
-                    var repeat_penalty = _params.repeat_penalty;
-                    var alpha_presence = _params.presence_penalty;
-                    var alpha_frequency = _params.frequency_penalty;
-                    var mirostat = _params.mirostat;
-                    var mirostat_tau = _params.mirostat_tau;
-                    var mirostat_eta = _params.mirostat_eta;
-                    var penalize_nl = _params.penalize_nl;
-
-                    // optionally save the session on first sample (for faster prompt loading next time)
-                    if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session)
-                    {
-                        _need_to_save_session = false;
-                        NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count);
-                    }
-
-                    llama_token id = 0;
-
-                    {
-                        var n_vocab = NativeApi.llama_n_vocab(_ctx);
-                        var logits = Utils.llama_get_logits(_ctx, n_vocab);
-
-                        // Apply params.logit_bias map
-                        foreach (var (key, value) in _params.logit_bias)
-                        {
-                            logits[key] += value;
-                        }
-
-                        var candidates = new List<LLamaTokenData>();
-                        candidates.Capacity = n_vocab;
-                        for (llama_token token_id = 0; token_id < n_vocab; token_id++)
-                        {
-                            candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f));
-                        }
-
-                        LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false);
-
-                        // Apply penalties
-                        float nl_logit = logits[NativeApi.llama_token_nl()];
-                        var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx);
-                        SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
-                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
-                            (ulong)last_n_repeat, repeat_penalty);
-                        SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
-                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
-                            (ulong)last_n_repeat, alpha_frequency, alpha_presence);
-                        if (!penalize_nl)
-                        {
-                            logits[NativeApi.llama_token_nl()] = nl_logit;
-                        }
-
-                        if (temp <= 0)
-                        {
-                            // Greedy sampling
-                            id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p);
-                        }
-                        else
-                        {
-                            if (mirostat == 1)
-                            {
-                                float mirostat_mu = 2.0f * mirostat_tau;
-                                const int mirostat_m = 100;
-                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
-                                id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, ref mirostat_mu);
-                            }
-                            else if (mirostat == 2)
-                            {
-                                float mirostat_mu = 2.0f * mirostat_tau;
-                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
-                                id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, ref mirostat_mu);
-                            }
-                            else
-                            {
-                                // Temperature sampling
-                                SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1);
-                                SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1);
-                                SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1);
-                                SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1);
-                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
-                                id = SamplingApi.llama_sample_token(_ctx, candidates_p);
-                            }
-                        }
-
-                        _last_n_tokens.RemoveAt(0);
-                        _last_n_tokens.Add(id);
-                    }
-
-                    // replace end of text token with newline token when in interactive mode
-                    if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct)
-                    {
-                        id = _llama_token_newline[0];
-                        if (_params.antiprompt.Count != 0)
-                        {
-                            // tokenize and inject first reverse prompt
-                            var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false, encoding);
-                            _embed_inp.AddRange(first_antiprompt);
-                        }
-                    }
-
-                    // add it to the context
-                    _embed.Add(id);
-
-                    // echo this to console
-                    _input_echo = true;
-
-                    // decrement remaining sampling budget
-                    _n_remain--;
-                }
-                else
-                {
-                    while (_embed_inp.Count > _n_consumed)
-                    {
-                        _embed.Add(_embed_inp[_n_consumed]);
-                        _last_n_tokens.RemoveAt(0);
-                        _last_n_tokens.Add(_embed_inp[_n_consumed]);
-                        _n_consumed++;
-                        if (_embed.Count >= _params.n_batch)
-                        {
-                            break;
-                        }
-                    }
-                }
-
-                if (_input_echo && !_is_interacting)
-                {
-                    foreach (var id in _embed)
-                    {
-                        var res = Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
-                        yield return res;
-                    }
-                }
-
-                if (_params.interactive && _embed_inp.Count <= _n_consumed)
-                {
-                    if (_params.antiprompt.Count > 0)
-                    {
-                        string last_output = "";
-                        foreach (var id in _last_n_tokens)
-                        {
-                            last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
-                        }
-
-                        _is_antiprompt = false;
-                        foreach (var antiprompt in _params.antiprompt)
-                        {
-                            if (last_output.EndsWith(antiprompt))
-                            {
-                                _is_interacting = true;
-                                _is_antiprompt = true;
-                                break;
-                            }
-                        }
-                    }
-
-                    if (_n_past > 0 && _is_interacting)
-                    {
-                        if (_params.instruct)
-                        {
-                            yield return "\n> ";
-                        }
-                        _input_echo = false;
-                        break;
-                    }
-
-                    if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos())
-                    {
-                        if (_params.instruct)
-                        {
-                            _is_interacting = true;
-                        }
-                        else
-                        {
-                            LLamaLogger.Default.Info(" [end of text]");
-                        }
-                    }
-
-                    if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1)
-                    {
-                        _n_remain = _params.n_predict;
-                        _is_interacting = true;
-                    }
+                    _logger.Error($"Failed to eval.");
+                    throw new RuntimeError("Failed to eval.");
                 }
-            }
 
-            if (!string.IsNullOrEmpty(_path_session) && _params.prompt_cache_all)
-            {
-                LLamaLogger.Default.Info($"saving final output to session file {_path_session}");
-                var session_token_array = _session_tokens.ToArray();
-                NativeApi.llama_save_session_file(_ctx, _path_session, session_token_array, (ulong)session_token_array.Length);
+                pastTokensCount += n_eval;
             }
+            return pastTokensCount;
         }
 
-        public void Dispose()
+        // TODO: add comment
+        internal IEnumerable<string> GenerateResult(IEnumerable<llama_token> ids)
         {
-            _ctx.Dispose();
+            foreach(var id in ids)
+            {
+                yield return Utils.TokenToString(id, _ctx, _encoding);
+            }
         }
     }
 }

LLama/LLamaSharp.csproj

@@ -70,4 +70,10 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Abstractions\ChatCompletion\" />
+    <Folder Include="Abstractions\Embeddings\" />
+    <Folder Include="Abstractions\TextCompletion\" />
+  </ItemGroup>
+
 </Project>

LLama/Native/LLamaTokenData.cs

@@ -6,7 +6,7 @@ using System.Text;
 namespace LLama.Native
 {
     [StructLayout(LayoutKind.Sequential)]
-    internal struct LLamaTokenData
+    public struct LLamaTokenData
     {
         /// <summary>
         /// token id

LLama/Native/LLamaTokenDataArray.cs

@@ -7,7 +7,7 @@ using System.Text;
 namespace LLama.Native
 {
     [StructLayout(LayoutKind.Sequential)]
-    internal struct LLamaTokenDataArray
+    public struct LLamaTokenDataArray
     {
         public Memory<LLamaTokenData> data;
         public ulong size;
@@ -23,7 +23,7 @@ namespace LLama.Native
     }
 
     [StructLayout(LayoutKind.Sequential)]
-    internal struct LLamaTokenDataArrayNative
+    public struct LLamaTokenDataArrayNative
     {
         public IntPtr data;
         public ulong size;

LLama/Native/NativeApi.cs

@@ -164,6 +164,9 @@ namespace LLama.Native
         [DllImport(libraryName)]
         public static extern int llama_eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int n_tokens, int n_past, int n_threads);
 
+        [DllImport(libraryName, EntryPoint = "llama_eval")]
+        public static extern int llama_eval_with_pointer(SafeLLamaContextHandle ctx, llama_token* tokens, int n_tokens, int n_past, int n_threads);
+
         /// <summary>
         /// Convert the provided text into tokens.
         /// The tokens pointer must be large enough to hold the resulting tokens.

LLama/Old/ChatSession.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace LLama.Old
+{
+    public class ChatSession<T> where T : IChatModel
+    {
+        IChatModel _model;
+        List<ChatMessageRecord> History { get; } = new List<ChatMessageRecord>();
+
+        public ChatSession(T model)
+        {
+            _model = model;
+        }
+
+        public IEnumerable<string> Chat(string text, string? prompt = null, string encoding = "UTF-8")
+        {
+            History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Human, text), DateTime.Now));
+            string totalResponse = "";
+            foreach (var response in _model.Chat(text, prompt, encoding))
+            {
+                totalResponse += response;
+                yield return response;
+            }
+            History.Add(new ChatMessageRecord(new ChatCompletionMessage(ChatRole.Assistant, totalResponse), DateTime.Now));
+        }
+
+        public ChatSession<T> WithPrompt(string prompt, string encoding = "UTF-8")
+        {
+            _model.InitChatPrompt(prompt, encoding);
+            return this;
+        }
+
+        public ChatSession<T> WithPromptFile(string promptFilename, string encoding = "UTF-8")
+        {
+            return WithPrompt(File.ReadAllText(promptFilename), encoding);
+        }
+
+        /// <summary>
+        /// Set the keyword to split the return value of chat AI.
+        /// </summary>
+        /// <param name="humanName"></param>
+        /// <returns></returns>
+        public ChatSession<T> WithAntiprompt(string[] antiprompt)
+        {
+            _model.InitChatAntiprompt(antiprompt);
+            return this;
+        }
+    }
+}

LLama/IChatModel.cs → LLama/Old/IChatModel.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace LLama
+namespace LLama.Old
 {
     public interface IChatModel
     {

LLama/LLamaEmbedder.cs → LLama/Old/LLamaEmbedder.cs

@@ -4,9 +4,9 @@ using System.Collections.Generic;
 using System.Text;
 using LLama.Exceptions;
 
-namespace LLama
+namespace LLama.Old
 {
-    public class LLamaEmbedder: IDisposable
+    public class LLamaEmbedder : IDisposable
     {
         SafeLLamaContextHandle _ctx;
 
@@ -27,7 +27,7 @@ namespace LLama
 
         public unsafe float[] GetEmbeddings(string text, int n_thread = -1, bool add_bos = true, string encoding = "UTF-8")
         {
-            if(n_thread == -1)
+            if (n_thread == -1)
             {
                 n_thread = Math.Max(Environment.ProcessorCount / 2, 1);
             }
@@ -51,7 +51,7 @@ namespace LLama
 
             int n_embed = NativeApi.llama_n_embd(_ctx);
             var embeddings = NativeApi.llama_get_embeddings(_ctx);
-            if(embeddings == null)
+            if (embeddings == null)
             {
                 return new float[0];
             }

LLama/Old/LLamaModel.cs

@@ -0,0 +1,804 @@
+using LLama.Exceptions;
+using LLama.Types;
+using LLama.Extensions;
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace LLama.Old
+{
+    using llama_token = Int32;
+    public class LLamaModel : IChatModel, IDisposable
+    {
+        LLamaParams _params;
+        SafeLLamaContextHandle _ctx;
+        string _path_session;
+        List<llama_token> _session_tokens;
+        List<llama_token> _embed_inp;
+        int _n_ctx;
+        List<llama_token> _inp_pfx;
+        List<llama_token> _inp_sfx;
+        List<llama_token> _llama_token_newline;
+        List<llama_token> _last_n_tokens;
+        bool _is_interacting;
+        bool _is_antiprompt;
+        bool _input_echo;
+        bool _verbose;
+
+        // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session
+        // if we loaded a session with at least 75% similarity. It's currently just used to speed up the
+        // initial prompt so it doesn't need to be an exact match.
+        bool _need_to_save_session;
+        int _n_past;
+        int _n_remain;
+        int _n_consumed;
+        int _n_session_consumed;
+        List<llama_token> _embed;
+
+        public string Name { get; set; }
+        public bool Verbose
+        {
+            get
+            {
+                return _verbose;
+            }
+            set
+            {
+                _verbose = value;
+            }
+        }
+        public SafeLLamaContextHandle NativeHandle => _ctx;
+
+        /// <summary>
+        /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will 
+        /// load 20 layers to gpu by default.
+        /// </summary>
+        /// <param name="model_path">The model file path.</param>
+        /// <param name="model_name">The model name.</param>
+        /// <param name="verbose">Whether to print details when running the model.</param>
+        /// <param name="seed"></param>
+        /// <param name="n_threads"></param>
+        /// <param name="n_predict"></param>
+        /// <param name="n_ctx"></param>
+        /// <param name="n_batch"></param>
+        /// <param name="n_keep"></param>
+        /// <param name="n_gpu_layers"></param>
+        /// <param name="logit_bias"></param>
+        /// <param name="top_k"></param>
+        /// <param name="top_p"></param>
+        /// <param name="tfs_z"></param>
+        /// <param name="typical_p"></param>
+        /// <param name="temp"></param>
+        /// <param name="repeat_penalty"></param>
+        /// <param name="repeat_last_n"></param>
+        /// <param name="frequency_penalty"></param>
+        /// <param name="presence_penalty"></param>
+        /// <param name="mirostat"></param>
+        /// <param name="mirostat_tau"></param>
+        /// <param name="mirostat_eta"></param>
+        /// <param name="prompt"></param>
+        /// <param name="path_session"></param>
+        /// <param name="input_prefix"></param>
+        /// <param name="input_suffix"></param>
+        /// <param name="antiprompt"></param>
+        /// <param name="lora_adapter"></param>
+        /// <param name="lora_base"></param>
+        /// <param name="memory_f16"></param>
+        /// <param name="random_prompt"></param>
+        /// <param name="use_color"></param>
+        /// <param name="interactive"></param>
+        /// <param name="embedding"></param>
+        /// <param name="interactive_first"></param>
+        /// <param name="prompt_cache_all"></param>
+        /// <param name="instruct"></param>
+        /// <param name="penalize_nl"></param>
+        /// <param name="perplexity"></param>
+        /// <param name="use_mmap"></param>
+        /// <param name="use_mlock"></param>
+        /// <param name="mem_test"></param>
+        /// <param name="verbose_prompt"></param>
+        /// <param name="encoding"></param>
+        public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1,
+            int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1,
+            Dictionary<llama_token, float> logit_bias = null, int top_k = 40, float top_p = 0.95f,
+            float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f,
+            int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f,
+            int mirostat = 0, float mirostat_tau = 5.00f, float mirostat_eta = 0.10f, string prompt = "",
+            string path_session = "", string input_prefix = "", string input_suffix = "",
+            List<string> antiprompt = null, string lora_adapter = "", string lora_base = "",
+            bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false,
+            bool embedding = false, bool interactive_first = false, bool prompt_cache_all = false, bool instruct = false, bool penalize_nl = true,
+            bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false,
+            bool verbose_prompt = false, string encoding = "UTF-8") : this(new LLamaParams(seed: seed,
+                n_threads: n_threads,
+                n_predict: n_predict,
+                n_ctx: n_ctx,
+                n_batch: n_batch,
+                n_keep: n_keep,
+                n_gpu_layers: n_gpu_layers,
+                logit_bias: logit_bias,
+                top_k: top_k,
+                top_p: top_p,
+                tfs_z: tfs_z,
+                typical_p: typical_p,
+                temp: temp,
+                repeat_penalty: repeat_penalty,
+                repeat_last_n: repeat_last_n,
+                frequency_penalty: frequency_penalty,
+                presence_penalty: presence_penalty,
+                mirostat: mirostat,
+                mirostat_tau: mirostat_tau,
+                mirostat_eta: mirostat_eta,
+                model: model_path,
+                prompt: prompt,
+                path_session: path_session,
+                input_prefix: input_prefix,
+                input_suffix: input_suffix,
+                antiprompt: antiprompt,
+                lora_adapter: lora_adapter,
+                lora_base: lora_base,
+                memory_f16: memory_f16,
+                random_prompt: random_prompt,
+                use_color: use_color,
+                interactive: interactive,
+                embedding: embedding,
+                interactive_first: interactive_first,
+                prompt_cache_all: prompt_cache_all,
+                instruct: instruct,
+                penalize_nl: penalize_nl,
+                perplexity: perplexity,
+                use_mmap: use_mmap,
+                use_mlock: use_mlock,
+                mem_test: mem_test,
+                verbose_prompt: verbose_prompt),
+                model_name, verbose, encoding)
+        {
+
+        }
+
+        /// <summary>
+        /// Please refer `LLamaParams` to find the meanings of each arg. Be sure to have set the `n_gpu_layers`, otherwise it will 
+        /// load 20 layers to gpu by default.
+        /// </summary>
+        /// <param name="params">The LLamaModel params</param>
+        /// <param name="name">Model name</param>
+        /// <param name="verbose">Whether to output the detailed info.</param>
+        /// <param name="encoding"></param>
+        /// <exception cref="RuntimeError"></exception>
+        public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8")
+        {
+            Name = name;
+            _params = @params;
+            _verbose = verbose;
+            _ctx = Utils.llama_init_from_gpt_params(ref _params);
+
+            // Add a space in front of the first character to match OG llama tokenizer behavior
+            _session_tokens = new List<llama_token>();
+
+            _path_session = @params.path_session;
+            if (!string.IsNullOrEmpty(_path_session))
+            {
+                if (verbose)
+                {
+                    LLamaLogger.Default.Info($"Attempting to load saved session from '{_path_session}'");
+                }
+
+                if (!File.Exists(_path_session))
+                {
+                    LLamaLogger.Default.Warn("Session file does not exist, will create.");
+                }
+
+                llama_token[] session_tokens = new llama_token[@params.n_ctx];
+                ulong n_token_count_out = 0;
+                if (!NativeApi.llama_load_session_file(_ctx, _path_session, session_tokens, (ulong)@params.n_ctx, &n_token_count_out))
+                {
+                    throw new RuntimeError($"Failed to load session file {_path_session}");
+                }
+                _session_tokens = session_tokens.Take((int)n_token_count_out).ToList();
+                if (verbose)
+                {
+                    LLamaLogger.Default.Info($"Loaded a session with prompt size of {_session_tokens.Count} tokens");
+                }
+            }
+
+            _n_ctx = NativeApi.llama_n_ctx(_ctx);
+
+            WithPrompt(_params.prompt);
+
+            // prefix & suffix for instruct mode
+            _inp_pfx = Utils.llama_tokenize(_ctx, "\n\n### Instruction:\n\n", true, encoding);
+            _inp_sfx = Utils.llama_tokenize(_ctx, "\n\n### Response:\n\n", false, encoding);
+
+            // in instruct mode, we inject a prefix and a suffix to each input by the user
+            if (_params.instruct)
+            {
+                _params.interactive_first = true;
+                _params.antiprompt.Add("### Instruction:\n\n");
+            }
+
+            // enable interactive mode if reverse prompt or interactive start is specified
+            if (_params.interactive_first)
+            {
+                _params.interactive = true;
+            }
+
+            // determine newline token
+            _llama_token_newline = Utils.llama_tokenize(_ctx, "\n", false, encoding);
+
+            if (_params.verbose_prompt)
+            {
+                LLamaLogger.Default.Info("\n");
+                LLamaLogger.Default.Info($"prompt: '{_params.prompt}'");
+                LLamaLogger.Default.Info($"number of tokens in prompt = {_embed_inp.Count}");
+                for (int i = 0; i < _embed_inp.Count; i++)
+                {
+                    LLamaLogger.Default.Info($"{_embed_inp[i]} -> '{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}'");
+                }
+                if (_params.n_keep > 0)
+                {
+                    LLamaLogger.Default.Info($"static prompt based on n_keep: '");
+                    for (int i = 0; i < _params.n_keep; i++)
+                    {
+                        LLamaLogger.Default.Info($"{NativeApi.llama_token_to_str(_ctx, _embed_inp[i])}");
+                    }
+                    LLamaLogger.Default.Info("\n");
+                }
+                LLamaLogger.Default.Info("\n");
+            }
+
+            if (_params.interactive && verbose)
+            {
+                LLamaLogger.Default.Info("interactive mode on.");
+            }
+            if (verbose)
+            {
+                LLamaLogger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " +
+                    $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " +
+                    $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," +
+                    $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," +
+                    $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}");
+                LLamaLogger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " +
+                    $"n_keep = {_params.n_keep}");
+                LLamaLogger.Default.Info("\n");
+            }
+
+            _last_n_tokens = Enumerable.Repeat(0, _n_ctx).ToList();
+
+            if (_params.interactive)
+            {
+                if (verbose)
+                {
+                    LLamaLogger.Default.Info("== Running in interactive mode. ==");
+                }
+                _is_interacting = _params.interactive_first;
+            }
+
+            _is_antiprompt = false;
+            _input_echo = false;
+            _n_past = 0;
+            _n_remain = _params.n_predict;
+            _n_consumed = 0;
+            _n_session_consumed = 0;
+            _embed = new List<llama_token>();
+        }
+
+        /// <summary>
+        /// Apply a prompt to the model.
+        /// </summary>
+        /// <param name="prompt"></param>
+        /// <param name="encoding"></param>
+        /// <returns></returns>
+        /// <exception cref="ArgumentException"></exception>
+        public LLamaModel WithPrompt(string prompt, string encoding = "UTF-8")
+        {
+            _params.prompt = prompt.Insert(0, " ");
+            _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true, encoding);
+
+            if (_embed_inp.Count > _n_ctx - 4)
+            {
+                throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})");
+            }
+
+            ulong n_matching_session_tokens = 0;
+            if (_session_tokens.Count > 0)
+            {
+                foreach (var id in _session_tokens)
+                {
+                    if (n_matching_session_tokens >= (ulong)_embed_inp.Count || id != _embed_inp[(int)n_matching_session_tokens])
+                    {
+                        break;
+                    }
+                    n_matching_session_tokens++;
+                }
+                if (n_matching_session_tokens >= (ulong)_embed_inp.Count)
+                {
+                    LLamaLogger.Default.Info("Session file has exact match for prompt!");
+                }
+                else if (n_matching_session_tokens < (ulong)(_embed_inp.Count / 2))
+                {
+                    LLamaLogger.Default.Warn($"session file has low similarity to prompt ({n_matching_session_tokens} " +
+                        $"/ {_embed_inp.Count} tokens); will mostly be reevaluated.");
+                }
+                else
+                {
+                    LLamaLogger.Default.Info($"Session file matches {n_matching_session_tokens} / {_embed_inp.Count} " +
+                        $"tokens of prompt.");
+                }
+            }
+            // number of tokens to keep when resetting context
+            if (_params.n_keep < 0 || _params.n_keep > _embed_inp.Count || _params.instruct)
+            {
+                _params.n_keep = _embed_inp.Count;
+            }
+            if (_embed_inp.Count > _n_ctx - 4)
+            {
+                throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})");
+            }
+            _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4);
+
+            return this;
+        }
+
+        /// <summary>
+        /// Apply the prompt file to the model.
+        /// </summary>
+        /// <param name="promptFileName"></param>
+        /// <returns></returns>
+        public LLamaModel WithPromptFile(string promptFileName)
+        {
+            return WithPrompt(File.ReadAllText(promptFileName));
+        }
+
+        private void ProcessTextBeforeInfer(string text, string encoding)
+        {
+            if (!string.IsNullOrEmpty(_params.input_prefix))
+            {
+                text = _params.input_prefix + text;
+            }
+            //if (!text.EndsWith("\n"))
+            //{
+            //    text += "\n";
+            //}
+            if (text.Length > 1)
+            {
+                // append input suffix if any
+                if (!string.IsNullOrEmpty(_params.input_suffix))
+                {
+                    text += _params.input_suffix;
+                    //yield return _params.input_suffix;
+                }
+
+                // instruct mode: insert instruction prefix
+                if (_params.instruct && !_is_antiprompt)
+                {
+                    _n_consumed = _embed_inp.Count;
+                    _embed_inp.AddRange(_inp_pfx);
+                }
+
+                var line_inp = Utils.llama_tokenize(_ctx, text, false, encoding);
+                _embed_inp.AddRange(line_inp);
+
+                // instruct mode: insert response suffix
+                if (_params.instruct)
+                {
+                    _embed_inp.AddRange(_inp_sfx);
+                }
+
+                _n_remain -= line_inp.Count;
+            }
+        }
+
+        public void InitChatPrompt(string prompt, string encoding = "UTF-8")
+        {
+            WithPrompt(prompt);
+        }
+
+        public void InitChatAntiprompt(string[] antiprompt)
+        {
+            _params.antiprompt = antiprompt.ToList();
+        }
+
+        /// <summary>
+        /// Chat with the LLaMa model under interactive mode.
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="prompt"></param>
+        /// <param name="encoding"></param>
+        /// <returns></returns>
+        /// <exception cref="ArgumentException"></exception>
+        public IEnumerable<string> Chat(string text, string? prompt = null, string encoding = "UTF-8")
+        {
+            if (!_params.interactive)
+            {
+                throw new ArgumentException("The chat API could be only used under interactive model.");
+            }
+            _input_echo = false;
+            if (!string.IsNullOrEmpty(prompt))
+            {
+                WithPrompt(prompt);
+            }
+            return Call(text, encoding);
+        }
+
+        /// <summary>
+        /// Save the state to specified path.
+        /// </summary>
+        /// <param name="filename"></param>
+        public void SaveState(string filename)
+        {
+            var stateSize = NativeApi.llama_get_state_size(_ctx);
+            byte[] stateMemory = new byte[stateSize];
+            NativeApi.llama_copy_state_data(_ctx, stateMemory);
+            File.WriteAllBytes(filename, stateMemory);
+        }
+
+        /// <summary>
+        /// Load the state from specified path.
+        /// </summary>
+        /// <param name="filename"></param>
+        /// <param name="clearPreviousEmbed">Whether to clear previous footprints of this model.</param>
+        /// <exception cref="RuntimeError"></exception>
+        public void LoadState(string filename, bool clearPreviousEmbed = true)
+        {
+            var stateMemory = File.ReadAllBytes(filename);
+            int stateSize = (int)NativeApi.llama_get_state_size(_ctx);
+            if (stateMemory.Length != stateSize)
+            {
+                throw new RuntimeError("Failed to validate state size.");
+            }
+            NativeApi.llama_set_state_data(_ctx, stateMemory);
+
+            if (clearPreviousEmbed)
+            {
+                WithPrompt(_params.prompt);
+            }
+        }
+
+        /// <summary>
+        /// Tokenize a string.
+        /// </summary>
+        /// <param name="text">The utf-8 encoded string to tokenize.</param>
+        /// <returns>A list of tokens.</returns>
+        /// <exception cref="RuntimeError">If the tokenization failed.</exception>
+        public List<llama_token> Tokenize(string text, string encoding = "UTF-8")
+        {
+            Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero);
+            var n_ctx = NativeApi.llama_n_ctx(_ctx);
+            var tokens = new llama_token[n_ctx];
+            var n_tokens = NativeApi.llama_tokenize(_ctx, text, Encoding.GetEncoding(encoding), tokens, n_ctx, true);
+            if (n_tokens < 0)
+            {
+                throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}");
+            }
+            return tokens.Take(n_tokens).ToList();
+        }
+
+        /// <summary>
+        /// Detokenize a list of tokens.
+        /// </summary>
+        /// <param name="tokens">The list of tokens to detokenize.</param>
+        /// <returns>The detokenized string.</returns>
+        public string DeTokenize(IEnumerable<llama_token> tokens)
+        {
+            Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero);
+            string output = "";
+            foreach (var token in tokens)
+            {
+                output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token));
+            }
+            return output;
+        }
+
+        /// <summary>
+        /// Call the model to run inference.
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="encoding"></param>
+        /// <returns></returns>
+        /// <exception cref="RuntimeError"></exception>
+        public IEnumerable<string> Call(string text, string encoding = "UTF-8")
+        {
+            _is_antiprompt = false;
+            if (_n_past > 0)
+            {
+                _is_interacting = false;
+            }
+            if (_is_interacting)
+            {
+                if (_verbose)
+                {
+                    LLamaLogger.Default.Warn("In interacting when calling the model, automatically changed it.");
+                }
+                _is_interacting = false;
+            }
+            ProcessTextBeforeInfer(text, encoding);
+
+            while ((_n_remain != 0 || _params.interactive) && !_is_interacting)
+            {
+                if (_embed.Count > 0)
+                {
+                    // infinite text generation via context swapping
+                    // if we run out of context:
+                    // - take the n_keep first tokens from the original prompt (via n_past)
+                    // - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches
+                    if (_n_past + _embed.Count > _n_ctx)
+                    {
+                        int n_left = _n_past - _params.n_keep;
+
+                        _n_past = Math.Max(1, _params.n_keep);
+
+                        // insert n_left/2 tokens at the start of embed from last_n_tokens
+                        _embed.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embed.Count).Skip(_n_ctx - n_left / 2 - _embed.Count));
+
+                        // stop saving session if we run out of context
+                        _path_session = "";
+                    }
+
+                    // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
+                    // REVIEW
+                    if (_n_session_consumed < _session_tokens.Count)
+                    {
+                        int i = 0;
+                        for (; i < _embed.Count; i++)
+                        {
+                            if (_embed[i] != _session_tokens[_n_session_consumed])
+                            {
+                                _session_tokens = _session_tokens.Take(_n_session_consumed).ToList();
+                                break;
+                            }
+
+                            _n_past++;
+                            _n_session_consumed++;
+
+                            if (_n_session_consumed >= _session_tokens.Count)
+                            {
+                                i++;
+                                break;
+                            }
+                        }
+
+                        if (i > 0)
+                        {
+                            _embed.RemoveRange(0, i);
+                        }
+                    }
+
+                    // evaluate tokens in batches
+                    // embed is typically prepared beforehand to fit within a batch, but not always
+                    for (int i = 0; i < _embed.Count; i += _params.n_batch)
+                    {
+                        int n_eval = _embed.Count - i;
+
+                        if (n_eval > _params.n_batch)
+                        {
+                            n_eval = _params.n_batch;
+                        }
+
+                        var array = _embed.Skip(i).ToArray();
+                        if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0)
+                        {
+                            LLamaLogger.Default.Error($"Failed to eval.");
+                            throw new RuntimeError("Failed to eval.");
+                        }
+
+                        _n_past += n_eval;
+                    }
+
+                    if (_embed.Count > 0 && !string.IsNullOrEmpty(_path_session))
+                    {
+                        _session_tokens.AddRange(_embed);
+                        _n_session_consumed = _session_tokens.Count;
+                    }
+                }
+
+                _embed.Clear();
+
+                if (_embed_inp.Count <= _n_consumed && !_is_interacting)
+                {
+                    var temp = _params.temp;
+                    var top_k = _params.top_k <= 0 ? NativeApi.llama_n_vocab(_ctx) : _params.top_k;
+                    var top_p = _params.top_p;
+                    var tfs_z = _params.tfs_z;
+                    var typical_p = _params.typical_p;
+                    var repeat_last_n = _params.repeat_last_n < 0 ? _n_ctx : _params.repeat_last_n;
+                    var repeat_penalty = _params.repeat_penalty;
+                    var alpha_presence = _params.presence_penalty;
+                    var alpha_frequency = _params.frequency_penalty;
+                    var mirostat = _params.mirostat;
+                    var mirostat_tau = _params.mirostat_tau;
+                    var mirostat_eta = _params.mirostat_eta;
+                    var penalize_nl = _params.penalize_nl;
+
+                    // optionally save the session on first sample (for faster prompt loading next time)
+                    if (!string.IsNullOrEmpty(_path_session) && _need_to_save_session)
+                    {
+                        _need_to_save_session = false;
+                        NativeApi.llama_save_session_file(_ctx, _path_session, _session_tokens.ToArray(), (ulong)_session_tokens.Count);
+                    }
+
+                    llama_token id = 0;
+
+                    {
+                        var n_vocab = NativeApi.llama_n_vocab(_ctx);
+                        var logits = Utils.llama_get_logits(_ctx, n_vocab);
+
+                        // Apply params.logit_bias map
+                        foreach (var (key, value) in _params.logit_bias)
+                        {
+                            logits[key] += value;
+                        }
+
+                        var candidates = new List<LLamaTokenData>();
+                        candidates.Capacity = n_vocab;
+                        for (llama_token token_id = 0; token_id < n_vocab; token_id++)
+                        {
+                            candidates.Add(new LLamaTokenData(token_id, logits[token_id], 0.0f));
+                        }
+
+                        LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates.ToArray(), (ulong)candidates.Count, false);
+
+                        // Apply penalties
+                        float nl_logit = logits[NativeApi.llama_token_nl()];
+                        var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx);
+                        SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
+                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
+                            (ulong)last_n_repeat, repeat_penalty);
+                        SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
+                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
+                            (ulong)last_n_repeat, alpha_frequency, alpha_presence);
+                        if (!penalize_nl)
+                        {
+                            logits[NativeApi.llama_token_nl()] = nl_logit;
+                        }
+
+                        if (temp <= 0)
+                        {
+                            // Greedy sampling
+                            id = SamplingApi.llama_sample_token_greedy(_ctx, candidates_p);
+                        }
+                        else
+                        {
+                            if (mirostat == 1)
+                            {
+                                float mirostat_mu = 2.0f * mirostat_tau;
+                                const int mirostat_m = 100;
+                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
+                                id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, ref mirostat_mu);
+                            }
+                            else if (mirostat == 2)
+                            {
+                                float mirostat_mu = 2.0f * mirostat_tau;
+                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
+                                id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, ref mirostat_mu);
+                            }
+                            else
+                            {
+                                // Temperature sampling
+                                SamplingApi.llama_sample_top_k(_ctx, candidates_p, top_k, 1);
+                                SamplingApi.llama_sample_tail_free(_ctx, candidates_p, tfs_z, 1);
+                                SamplingApi.llama_sample_typical(_ctx, candidates_p, typical_p, 1);
+                                SamplingApi.llama_sample_top_p(_ctx, candidates_p, top_p, 1);
+                                SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
+                                id = SamplingApi.llama_sample_token(_ctx, candidates_p);
+                            }
+                        }
+
+                        _last_n_tokens.RemoveAt(0);
+                        _last_n_tokens.Add(id);
+                    }
+
+                    // replace end of text token with newline token when in interactive mode
+                    if (id == NativeApi.llama_token_eos() && _params.interactive && !_params.instruct)
+                    {
+                        id = _llama_token_newline[0];
+                        if (_params.antiprompt.Count != 0)
+                        {
+                            // tokenize and inject first reverse prompt
+                            var first_antiprompt = Utils.llama_tokenize(_ctx, _params.antiprompt[0], false, encoding);
+                            _embed_inp.AddRange(first_antiprompt);
+                        }
+                    }
+
+                    // add it to the context
+                    _embed.Add(id);
+
+                    // echo this to console
+                    _input_echo = true;
+
+                    // decrement remaining sampling budget
+                    _n_remain--;
+                }
+                else
+                {
+                    while (_embed_inp.Count > _n_consumed)
+                    {
+                        _embed.Add(_embed_inp[_n_consumed]);
+                        _last_n_tokens.RemoveAt(0);
+                        _last_n_tokens.Add(_embed_inp[_n_consumed]);
+                        _n_consumed++;
+                        if (_embed.Count >= _params.n_batch)
+                        {
+                            break;
+                        }
+                    }
+                }
+
+                if (_input_echo && !_is_interacting)
+                {
+                    foreach (var id in _embed)
+                    {
+                        var res = Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
+                        yield return res;
+                    }
+                }
+
+                if (_params.interactive && _embed_inp.Count <= _n_consumed)
+                {
+                    if (_params.antiprompt.Count > 0)
+                    {
+                        string last_output = "";
+                        foreach (var id in _last_n_tokens)
+                        {
+                            last_output += Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
+                        }
+
+                        _is_antiprompt = false;
+                        foreach (var antiprompt in _params.antiprompt)
+                        {
+                            if (last_output.EndsWith(antiprompt))
+                            {
+                                _is_interacting = true;
+                                _is_antiprompt = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (_n_past > 0 && _is_interacting)
+                    {
+                        if (_params.instruct)
+                        {
+                            yield return "\n> ";
+                        }
+                        _input_echo = false;
+                        break;
+                    }
+
+                    if (_embed.Count > 0 && _embed.Last() == NativeApi.llama_token_eos())
+                    {
+                        if (_params.instruct)
+                        {
+                            _is_interacting = true;
+                        }
+                        else
+                        {
+                            LLamaLogger.Default.Info(" [end of text]");
+                        }
+                    }
+
+                    if (_params.interactive && _n_remain <= 0 && _params.n_predict != -1)
+                    {
+                        _n_remain = _params.n_predict;
+                        _is_interacting = true;
+                    }
+                }
+            }
+
+            if (!string.IsNullOrEmpty(_path_session) && _params.prompt_cache_all)
+            {
+                LLamaLogger.Default.Info($"saving final output to session file {_path_session}");
+                var session_token_array = _session_tokens.ToArray();
+                NativeApi.llama_save_session_file(_ctx, _path_session, session_token_array, (ulong)session_token_array.Length);
+            }
+        }
+
+        public void Dispose()
+        {
+            _ctx.Dispose();
+        }
+    }
+}

LLama/LLamaParams.cs → LLama/Old/LLamaParams.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 
-namespace LLama
+namespace LLama.Old
 {
     using llama_token = Int32;
     public struct LLamaParams
@@ -66,7 +66,7 @@ namespace LLama
             string path_session = "", string input_prefix = "", string input_suffix = "",
             List<string> antiprompt = null, string lora_adapter = "", string lora_base = "",
             bool memory_f16 = true, bool random_prompt = false, bool use_color = false, bool interactive = false,
-            bool prompt_cache_all = false, bool embedding = false, bool interactive_first = false, 
+            bool prompt_cache_all = false, bool embedding = false, bool interactive_first = false,
             bool instruct = false, bool penalize_nl = true,
             bool perplexity = false, bool use_mmap = true, bool use_mlock = false, bool mem_test = false,
             bool verbose_prompt = false)

LLama/LLamaTypes.cs → LLama/Old/LLamaTypes.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace LLama.Types
+namespace LLama.Old
 {
     public enum ChatRole
     {

LLama/Old/Utils.cs

@@ -0,0 +1,98 @@
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using LLama.Exceptions;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.IO;
+
+namespace LLama.Old
+{
+    using llama_token = Int32;
+    internal static class Utils
+    {
+        public static SafeLLamaContextHandle llama_init_from_gpt_params(ref LLamaParams @params)
+        {
+            var lparams = NativeApi.llama_context_default_params();
+
+            lparams.n_ctx = @params.n_ctx;
+            lparams.n_gpu_layers = @params.n_gpu_layers;
+            lparams.seed = @params.seed;
+            lparams.f16_kv = @params.memory_f16;
+            lparams.use_mmap = @params.use_mmap;
+            lparams.use_mlock = @params.use_mlock;
+            lparams.logits_all = @params.perplexity;
+            lparams.embedding = @params.embedding;
+
+            if (!File.Exists(@params.model))
+            {
+                throw new FileNotFoundException($"The model file does not exist: {@params.model}");
+            }
+
+            var ctx_ptr = NativeApi.llama_init_from_file(@params.model, lparams);
+
+            if (ctx_ptr == IntPtr.Zero)
+            {
+                throw new RuntimeError($"Failed to load model {@params.model}.");
+            }
+
+            SafeLLamaContextHandle ctx = new(ctx_ptr);
+
+            if (!string.IsNullOrEmpty(@params.lora_adapter))
+            {
+                int err = NativeApi.llama_apply_lora_from_file(ctx, @params.lora_adapter,
+                    string.IsNullOrEmpty(@params.lora_base) ? null : @params.lora_base, @params.n_threads);
+                if (err != 0)
+                {
+                    throw new RuntimeError("Failed to apply lora adapter.");
+                }
+            }
+            return ctx;
+        }
+
+        public static List<llama_token> llama_tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, string encodingName)
+        {
+            var encoding = Encoding.GetEncoding(encodingName);
+            var cnt = encoding.GetByteCount(text);
+            llama_token[] res = new llama_token[cnt + (add_bos ? 1 : 0)];
+            int n = NativeApi.llama_tokenize(ctx, text, encoding, res, res.Length, add_bos);
+            if (n < 0)
+            {
+                throw new RuntimeError("Error happened during tokenization. It's possibly caused by wrong encoding. Please try to " +
+                    "specify the encoding.");
+            }
+            return res.Take(n).ToList();
+        }
+
+        public unsafe static Span<float> llama_get_logits(SafeLLamaContextHandle ctx, int length)
+        {
+            var logits = NativeApi.llama_get_logits(ctx);
+            return new Span<float>(logits, length);
+        }
+
+        public static unsafe string PtrToStringUTF8(IntPtr ptr)
+        {
+#if NET6_0_OR_GREATER
+            return Marshal.PtrToStringUTF8(ptr);
+#else
+            byte* tp = (byte*)ptr.ToPointer();
+            List<byte> bytes = new();
+            while (true)
+            {
+                byte c = *tp++;
+                if (c == '\0')
+                {
+                    break;
+                }
+                else
+                {
+                    bytes.Add(c);
+                }
+            }
+            return Encoding.UTF8.GetString(bytes.ToArray());
+#endif
+        }
+    }
+}

LLama/Utils.cs

@@ -1,50 +1,50 @@
-using LLama.Native;
+using LLama.Abstractions.Params;
+using LLama.Exceptions;
+using LLama.Native;
 using System;
 using System.Collections.Generic;
-using System.Text;
-using LLama.Exceptions;
-using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
-using System.IO;
+using System.Text;
 
 namespace LLama
 {
     using llama_token = Int32;
     internal static class Utils
     {
-        public static SafeLLamaContextHandle llama_init_from_gpt_params(ref LLamaParams @params)
+        public static SafeLLamaContextHandle InitLLamaContextFromModelParams(ModelParams @params)
         {
             var lparams = NativeApi.llama_context_default_params();
 
-            lparams.n_ctx = @params.n_ctx;
-            lparams.n_gpu_layers = @params.n_gpu_layers;
-            lparams.seed = @params.seed;
-            lparams.f16_kv = @params.memory_f16;
-            lparams.use_mmap = @params.use_mmap;
-            lparams.use_mlock = @params.use_mlock;
-            lparams.logits_all = @params.perplexity;
-            lparams.embedding = @params.embedding;
+            lparams.n_ctx = @params.ContextSize;
+            lparams.n_gpu_layers = @params.GpuLayerCount;
+            lparams.seed = @params.Seed;
+            lparams.f16_kv = @params.UseFp16Memory;
+            lparams.use_mmap = @params.UseMemoryLock;
+            lparams.use_mlock = @params.UseMemoryLock;
+            lparams.logits_all = @params.Perplexity;
+            lparams.embedding = @params.EmbeddingMode;
 
-            if (!File.Exists(@params.model))
+            if (!File.Exists(@params.ModelPath))
             {
-                throw new FileNotFoundException($"The model file does not exist: {@params.model}");
+                throw new FileNotFoundException($"The model file does not exist: {@params.ModelPath}");
             }
 
-            var ctx_ptr = NativeApi.llama_init_from_file(@params.model, lparams);
+            var ctx_ptr = NativeApi.llama_init_from_file(@params.ModelPath, lparams);
 
-            if(ctx_ptr == IntPtr.Zero )
+            if (ctx_ptr == IntPtr.Zero)
             {
-                throw new RuntimeError($"Failed to load model {@params.model}.");
+                throw new RuntimeError($"Failed to load model {@params.ModelPath}.");
             }
 
             SafeLLamaContextHandle ctx = new(ctx_ptr);
 
-            if (!string.IsNullOrEmpty(@params.lora_adapter))
+            if (!string.IsNullOrEmpty(@params.LoraAdapter))
             {
-                int err = NativeApi.llama_apply_lora_from_file(ctx, @params.lora_adapter,
-                    string.IsNullOrEmpty(@params.lora_base) ? null : @params.lora_base, @params.n_threads);
-                if(err != 0)
+                int err = NativeApi.llama_apply_lora_from_file(ctx, @params.LoraAdapter,
+                    string.IsNullOrEmpty(@params.LoraBase) ? null : @params.LoraBase, @params.Threads);
+                if (err != 0)
                 {
                     throw new RuntimeError("Failed to apply lora adapter.");
                 }
@@ -52,37 +52,62 @@ namespace LLama
             return ctx;
         }
 
-        public static List<llama_token> llama_tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, string encodingName)
+        public static IEnumerable<llama_token> Tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, Encoding encoding)
         {
-            var encoding = Encoding.GetEncoding(encodingName);
             var cnt = encoding.GetByteCount(text);
             llama_token[] res = new llama_token[cnt + (add_bos ? 1 : 0)];
             int n = NativeApi.llama_tokenize(ctx, text, encoding, res, res.Length, add_bos);
-            if(n < 0)
+            if (n < 0)
             {
                 throw new RuntimeError("Error happened during tokenization. It's possibly caused by wrong encoding. Please try to " +
                     "specify the encoding.");
             }
-            return res.Take(n).ToList();
+            return res.Take(n);
         }
 
-        public unsafe static Span<float> llama_get_logits(SafeLLamaContextHandle ctx, int length)
+        public unsafe static Span<float> GetLogits(SafeLLamaContextHandle ctx, int length)
         {
             var logits = NativeApi.llama_get_logits(ctx);
             return new Span<float>(logits, length);
         }
 
-        public static unsafe string PtrToStringUTF8(IntPtr ptr)
+        public static unsafe int Eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int startIndex, int n_tokens, int n_past, int n_threads)
+        {
+            int result;
+            fixed(llama_token* p = tokens)
+            {
+                result = NativeApi.llama_eval_with_pointer(ctx, p + startIndex, n_tokens, n_past, n_threads);
+            }
+            return result;
+        }
+
+        public static string TokenToString(llama_token token, SafeLLamaContextHandle ctx, Encoding encoding)
+        {
+            return PtrToString(NativeApi.llama_token_to_str(ctx, token), encoding);
+        }
+
+        public static unsafe string PtrToString(IntPtr ptr, Encoding encoding)
         {
 #if NET6_0_OR_GREATER
-            return Marshal.PtrToStringUTF8(ptr);
+            if(encoding == Encoding.UTF8)
+            {
+                return Marshal.PtrToStringUTF8(ptr);
+            }
+            else if(encoding == Encoding.Unicode)
+            {
+                return Marshal.PtrToStringUni(ptr);
+            }
+            else
+            {
+                return Marshal.PtrToStringAuto(ptr);
+            }
 #else
             byte* tp = (byte*)ptr.ToPointer();
             List<byte> bytes = new();
             while (true)
             {
                 byte c = *tp++;
-                if(c == '\0')
+                if (c == '\0')
                 {
                     break;
                 }
@@ -91,7 +116,7 @@ namespace LLama
                     bytes.Add(c);
                 }
             }
-            return Encoding.UTF8.GetString(bytes.ToArray());
+            return encoding.GetString(bytes.ToArray());
 #endif
         }
     }