refactor: some parts of code of LLamaModel.

LLama/Extensions/DictionaryExtensions.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Extensions
+{
+    public static class DictionaryExtension
+    {
+        public static void Deconstruct<T1, T2>(this KeyValuePair<T1, T2> pair, out T1 first, out T2 second)
+        {
+            first = pair.Key;
+            second = pair.Value;
+        }
+        public static void Update<T1, T2>(this Dictionary<T1, T2> dic, IDictionary<T1, T2> other)
+        {
+            foreach (var (key, value) in other)
+            {
+                dic[key] = value;
+            }
+        }
+        public static T2 GetOrDefault<T1, T2>(this Dictionary<T1, T2> dic, T1 key, T2 defaultValue)
+        {
+            if (dic.ContainsKey(key))
+            {
+                return dic[key];
+            }
+            return defaultValue;
+        }
+    }
+}

LLama/LLamaModel.cs

@@ -8,6 +8,7 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Runtime.InteropServices;
 using System.Diagnostics;
+using LLama.Extensions;
 
 namespace LLama
 {
@@ -182,7 +183,7 @@ namespace LLama
 
         public LLamaModel WithPrompt(string prompt, string encoding = "UTF-8")
         {
-            _params.prompt = _params.prompt.Insert(0, " ");
+            _params.prompt = prompt.Insert(0, " ");
             _embed_inp = Utils.llama_tokenize(_ctx, _params.prompt, true, encoding);
 
             if (_embed_inp.Count > _n_ctx - 4)
@@ -297,7 +298,11 @@ namespace LLama
 
         public IEnumerable<string> Call(string text, string encoding = "UTF-8")
         {
-            _is_interacting = _is_antiprompt = false;
+            _is_antiprompt = false;
+            if(_n_past > 0)
+            {
+                _is_interacting = false;
+            }
             ProcessTextBeforeInfer(text, encoding);
             
             while ((_n_remain != 0 || _params.interactive) && !_is_interacting)
@@ -336,9 +341,9 @@ namespace LLama
                         int i = 0;
                         for (; i < _embed.Count; i++)
                         {
-                            if (!_embed[i].Equals(_session_tokens[_n_session_consumed]))
+                            if (_embed[i] != _session_tokens[_n_session_consumed])
                             {
-                                _session_tokens.RemoveRange(_n_session_consumed, _session_tokens.Count - _n_session_consumed);
+                                _session_tokens = _session_tokens.Take(_n_session_consumed).ToList();
                                 break;
                             }
 
@@ -369,11 +374,11 @@ namespace LLama
                             n_eval = _params.n_batch;
                         }
 
-                        var array = _embed.GetRange(i, n_eval).ToArray();
+                        var array = _embed.Skip(i).ToArray();
                         if (NativeApi.llama_eval(_ctx, array, n_eval, _n_past, _params.n_threads) != 0)
                         {
-                            Logger.Default.Error($"Failed to eval");
-                            throw new RuntimeError("Failed to eval");
+                            Logger.Default.Error($"Failed to eval.");
+                            throw new RuntimeError("Failed to eval.");
                         }
 
                         _n_past += n_eval;
@@ -418,9 +423,9 @@ namespace LLama
                         var logits = Utils.llama_get_logits(_ctx, n_vocab);
 
                         // Apply params.logit_bias map
-                        foreach (KeyValuePair<int, float> it in _params.logit_bias)
+                        foreach (var (key, value) in _params.logit_bias)
                         {
-                            logits[it.Key] += it.Value;
+                            logits[key] += value;
                         }
 
                         var candidates = new List<LLamaTokenData>();
@@ -436,10 +441,10 @@ namespace LLama
                         float nl_logit = logits[NativeApi.llama_token_nl()];
                         var last_n_repeat = Math.Min(Math.Min(_last_n_tokens.Count, repeat_last_n), _n_ctx);
                         SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
-                            _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(),
+                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
                             (ulong)last_n_repeat, repeat_penalty);
                         SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
-                            _last_n_tokens.GetRange(_last_n_tokens.Count - last_n_repeat, last_n_repeat).ToArray(),
+                            _last_n_tokens.Skip(_last_n_tokens.Count - last_n_repeat).ToArray(),
                             (ulong)last_n_repeat, alpha_frequency, alpha_presence);
                         if (!penalize_nl)
                         {
@@ -458,13 +463,13 @@ namespace LLama
                                 float mirostat_mu = 2.0f * mirostat_tau;
                                 const int mirostat_m = 100;
                                 SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
-                                id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, mirostat_mu);
+                                id = SamplingApi.llama_sample_token_mirostat(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_m, ref mirostat_mu);
                             }
                             else if (mirostat == 2)
                             {
                                 float mirostat_mu = 2.0f * mirostat_tau;
                                 SamplingApi.llama_sample_temperature(_ctx, candidates_p, temp);
-                                id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, mirostat_mu);
+                                id = SamplingApi.llama_sample_token_mirostat_v2(_ctx, candidates_p, mirostat_tau, mirostat_eta, ref mirostat_mu);
                             }
                             else
                             {
@@ -521,11 +526,12 @@ namespace LLama
                     }
                 }
 
-                if (_input_echo)
+                if (_input_echo && !_is_interacting)
                 {
                     foreach (var id in _embed)
                     {
-                        yield return Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
+                        var res = Utils.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, id));
+                        yield return res;
                     }
                 }
 
@@ -553,6 +559,10 @@ namespace LLama
 
                     if(_n_past > 0 && _is_interacting)
                     {
+                        if (_params.instruct)
+                        {
+                            yield return "\n> ";
+                        }
                         _input_echo = false;
                         break;
                     }

LLama/Native/SamplingApi.cs

@@ -148,7 +148,7 @@ namespace LLama.Native
         /// <param name="m">The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.</param>
         /// <param name="mu">Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.</param>
         /// <returns></returns>
-        public static llama_token llama_sample_token_mirostat(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, int m, in float mu)
+        public static llama_token llama_sample_token_mirostat(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, int m, ref float mu)
         {
             var handle = candidates.data.Pin();
             var st = new LLamaTokenDataArrayNative();
@@ -172,7 +172,7 @@ namespace LLama.Native
         /// <param name="eta">The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.</param>
         /// <param name="mu">Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.</param>
         /// <returns></returns>
-        public static llama_token llama_sample_token_mirostat_v2(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, in float mu)
+        public static llama_token llama_sample_token_mirostat_v2(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, ref float mu)
         {
             var handle = candidates.data.Pin();
             var st = new LLamaTokenDataArrayNative();