refactor: instruct mode and examples.

LLama.Examples/ChatSession.cs

@@ -25,7 +25,7 @@ namespace LLama.Examples
                 Console.ForegroundColor = ConsoleColor.Green;
                 var question = Console.ReadLine();
                 Console.ForegroundColor = ConsoleColor.White;
-                var outputs = _session.Chat(question);
+                var outputs = _session.Chat(question, encoding: "UTF-8");
                 foreach (var output in outputs)
                 {
                     Console.Write(output);

LLama.Examples/ChatWithLLamaModel.cs

@@ -12,7 +12,7 @@ namespace LLama.Examples
         public ChatWithLLamaModel(string modelPath, string promptFilePath, string[] antiprompt)
         {
             _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 512, interactive: true, antiprompt: antiprompt.ToList(),
-                repeat_penalty: 1.0f), echo_input: false).WithPromptFile(promptFilePath);
+                repeat_penalty: 1.0f)).WithPromptFile(promptFilePath);
         }
 
         public void Run()

LLama.Examples/InstructMode.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace LLama.Examples
+{
+    public class InstructMode
+    {
+        LLamaModel _model;
+        public InstructMode(string modelPath, string promptFile)
+        {
+            _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 2048, n_predict: -1, top_k: 10000, instruct: true,
+                repeat_penalty: 1.1f, n_batch: 256, temp: 0.2f)).WithPromptFile(promptFile);
+        }
+
+        public void Run()
+        {
+            Console.WriteLine("\n### Instruction:\n >");
+            while (true)
+            {
+                Console.ForegroundColor = ConsoleColor.Green;
+                var question = Console.ReadLine();
+                Console.ForegroundColor = ConsoleColor.White;
+                var outputs = _model.Call(question);
+                foreach (var output in outputs)
+                {
+                    Console.Write(output);
+                }
+            }
+        }
+    }
+}

LLama.Examples/LLama.Examples.csproj

@@ -16,6 +16,24 @@
     <None Update="Assets\chat-with-bob.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="Assets\chat.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\alpaca.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\chat-with-vicuna-v0.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\chat-with-vicuna-v1.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\dan.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Assets\reason-act.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>

LLama.Examples/Program.cs

@@ -2,31 +2,87 @@
 using LLama.Examples;
 using LLama.Types;
 
-int choice = 0;
+Console.WriteLine("================LLamaSharp Examples==================\n");
 
-if(choice == 0)
-{
-    ChatSession chat = new(@"<Your model file path>", @"<Your prompt file path>", new string[] { "User:" });
-    chat.Run();
-}
-else if(choice == 1)
-{
-    ChatWithLLamaModel chat = new(@"<Your model file path>", "<Your prompt file path>", new string[] { "User:" });
-    chat.Run();
-}
-else if(choice == 2)
-{
-    ChatWithLLamaModelV1 chat = new(@"<Your model file path>");
-    chat.Run();
-}
-else if (choice == 3) // quantization
-{
-    Quantize q = new Quantize();
-    q.Run(@"<Your src model file path>",
-        @"<Your dst model file path>", "q4_1");
-}
-else if (choice == 4) // get the embeddings only
+Console.WriteLine("Please input a number to choose an example to run:");
+Console.WriteLine("0: Run a chat session.");
+Console.WriteLine("1: Run a LLamaModel to chat.");
+Console.WriteLine("2: Quantize a model.");
+Console.WriteLine("3: Get the embeddings of a message.");
+Console.WriteLine("4: Run a LLamaModel with instruct mode.");
+Console.WriteLine("5: Load and save state of LLamaModel.");
+
+
+while (true)
 {
-    GetEmbeddings em = new GetEmbeddings(@"<Your model file path>");
-    em.Run("Hello, what is python?");
+    Console.Write("\nYour choice: ");
+    int choice = int.Parse(Console.ReadLine());
+
+    if (choice == 0)
+    {
+        Console.Write("Please input your model path: ");
+        var modelPath = Console.ReadLine();
+        ChatSession chat = new(modelPath, "Assets/chat-with-bob.txt", new string[] { "User:" });
+        chat.Run();
+    }
+    else if (choice == 1)
+    {
+        Console.Write("Please input your model path: ");
+        var modelPath = Console.ReadLine();
+        ChatWithLLamaModel chat = new(modelPath, "Assets/chat-with-bob.txt", new string[] { "User:" });
+        chat.Run();
+    }
+    else if (choice == 2) // quantization
+    {
+        Console.Write("Please input your original model path: ");
+        var inputPath = Console.ReadLine();
+        Console.Write("Please input your output model path: ");
+        var outputPath = Console.ReadLine();
+        Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): ");
+        var quantizeType = Console.ReadLine();
+        Quantize q = new Quantize();
+        q.Run(inputPath, outputPath, quantizeType);
+    }
+    else if (choice == 3) // get the embeddings only
+    {
+        Console.Write("Please input your model path: ");
+        var modelPath = Console.ReadLine();
+        GetEmbeddings em = new GetEmbeddings(modelPath);
+        Console.Write("Please input the text: ");
+        var text = Console.ReadLine();
+        em.Run(text);
+    }
+    else if (choice == 4) // instruct mode
+    {
+        Console.Write("Please input your model path: ");
+        var modelPath = Console.ReadLine();
+        InstructMode im = new InstructMode(modelPath, "Assets/alpaca.txt");
+        Console.WriteLine("Here's a simple example for using instruct mode. You can input some words and let AI " +
+            "complete it for you. For example: Write a story about a fox that wants to make friend with human. No less than 200 words.");
+        im.Run();
+    }
+    else if (choice == 5) // load and save state
+    {
+        Console.Write("Please input your model path: ");
+        var modelPath = Console.ReadLine();
+        Console.Write("Please input your state file path: ");
+        var statePath = Console.ReadLine();
+        SaveAndLoadState sals = new(modelPath, File.ReadAllText(@"D:\development\llama\llama.cpp\prompts\alpaca.txt"));
+        sals.Run("Write a story about a fox that wants to make friend with human. No less than 200 words.");
+        sals.SaveState(statePath);
+        sals.Dispose();
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+
+        // create a new model to load the state.
+        SaveAndLoadState sals2 = new(modelPath, "");
+        sals2.LoadState(statePath);
+        sals2.Run("Tell me more things about the fox in the story you told me.");
+    }
+    else
+    {
+        Console.WriteLine("Cannot parse your choice. Please select again.");
+        continue;
+    }
+    break;
 }

LLama.Examples/SaveAndLoadState.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace LLama.Examples
+{
+    public class SaveAndLoadState: IDisposable
+    {
+        LLamaModel _model;
+        public SaveAndLoadState(string modelPath, string prompt)
+        {
+            _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 2048, n_predict: -1, top_k: 10000, instruct: true,
+                repeat_penalty: 1.1f, n_batch: 256, temp: 0.2f)).WithPrompt(prompt);
+        }
+
+        public void Run(string question)
+        {
+            // Only run once here.
+            Console.Write("\nUser:");
+            Console.ForegroundColor = ConsoleColor.Green;
+            Console.WriteLine(question);
+            Console.ForegroundColor = ConsoleColor.White;
+            var outputs = _model.Call(question);
+            foreach (var output in outputs)
+            {
+                Console.Write(output);
+            }
+        }
+
+        public void SaveState(string filename)
+        {
+            _model.SaveState(filename);
+            Console.WriteLine("Saved state!");
+        }
+
+        public void LoadState(string filename)
+        {
+            _model.LoadState(filename);
+            Console.WriteLine("Loaded state!");
+        }
+
+        public void Dispose()
+        {
+            _model.Dispose();
+        }
+    }
+}

LLama/LLamaModel.cs

@@ -25,6 +25,7 @@ namespace LLama
         bool _is_interacting;
         bool _is_antiprompt;
         bool _input_echo;
+        bool _verbose;
 
         // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session
         // if we loaded a session with at least 75% similarity. It's currently just used to speed up the
@@ -37,6 +38,17 @@ namespace LLama
         List<llama_token> _embed;
 
         public string Name { get; set; }
+        public bool Verbose
+        {
+            get
+            {
+                return _verbose;
+            }
+            set
+            {
+                _verbose = value;
+            }
+        }
         public SafeLLamaContextHandle NativeHandle => _ctx;
 
         /// <summary>
@@ -44,7 +56,6 @@ namespace LLama
         /// </summary>
         /// <param name="model_path">The model file path.</param>
         /// <param name="model_name">The model name.</param>
-        /// <param name="echo_input">Whether to print the input messages.</param>
         /// <param name="verbose">Whether to print details when running the model.</param>
         /// <param name="seed"></param>
         /// <param name="n_threads"></param>
@@ -88,7 +99,7 @@ namespace LLama
         /// <param name="mem_test"></param>
         /// <param name="verbose_prompt"></param>
         /// <param name="encoding"></param>
-        public LLamaModel(string model_path, string model_name, bool echo_input = false, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1,
+        public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1,
             int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1,
             Dictionary<llama_token, float> logit_bias = null, int top_k = 40, float top_p = 0.95f,
             float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f,
@@ -141,15 +152,24 @@ namespace LLama
                 use_mlock: use_mlock,
                 mem_test: mem_test,
                 verbose_prompt: verbose_prompt),
-                model_name, echo_input, verbose, encoding)
+                model_name, verbose, encoding)
         {
 
         }
 
-        public unsafe LLamaModel(LLamaParams @params, string name = "", bool echo_input = false, bool verbose = false, string encoding = "UTF-8")
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="params">The LLamaModel params</param>
+        /// <param name="name">Model name</param>
+        /// <param name="verbose">Whether to output the detailed info.</param>
+        /// <param name="encoding"></param>
+        /// <exception cref="RuntimeError"></exception>
+        public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8")
         {
             Name = name;
             _params = @params;
+            _verbose = verbose;
             _ctx = Utils.llama_init_from_gpt_params(ref _params);
 
             // Add a space in front of the first character to match OG llama tokenizer behavior
@@ -197,7 +217,7 @@ namespace LLama
             }
 
             // enable interactive mode if reverse prompt or interactive start is specified
-            if (_params.antiprompt.Count != 0 || _params.interactive_first)
+            if (_params.interactive_first)
             {
                 _params.interactive = true;
             }
@@ -233,10 +253,10 @@ namespace LLama
             if (verbose)
             {
                 Logger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " +
-                $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " +
-                $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," +
-                $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," +
-                $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}");
+                    $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " +
+                    $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," +
+                    $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," +
+                    $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}");
                 Logger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " +
                     $"n_keep = {_params.n_keep}");
                 Logger.Default.Info("\n");
@@ -254,7 +274,7 @@ namespace LLama
             }
 
             _is_antiprompt = false;
-            _input_echo = echo_input;
+            _input_echo = false;
             _n_past = 0;
             _n_remain = _params.n_predict;
             _n_consumed = 0;
@@ -315,6 +335,7 @@ namespace LLama
                 throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})");
             }
             _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4);
+            
             return this;
         }
 
@@ -328,23 +349,23 @@ namespace LLama
             return WithPrompt(File.ReadAllText(promptFileName));
         }
 
-        private string ProcessTextBeforeInfer(string text, string encoding)
+        private void ProcessTextBeforeInfer(string text, string encoding)
         {
             if (!string.IsNullOrEmpty(_params.input_prefix))
             {
                 text = _params.input_prefix + text;
             }
-            if (!text.EndsWith("\n"))
-            {
-                text += "\n";
-            }
+            //if (!text.EndsWith("\n"))
+            //{
+            //    text += "\n";
+            //}
             if (text.Length > 1)
             {
                 // append input suffix if any
                 if (!string.IsNullOrEmpty(_params.input_suffix))
                 {
                     text += _params.input_suffix;
-                    Console.Write(_params.input_suffix);
+                    //yield return _params.input_suffix;
                 }
 
                 // instruct mode: insert instruction prefix
@@ -365,7 +386,6 @@ namespace LLama
 
                 _n_remain -= line_inp.Count;
             }
-            return text;
         }
 
         public void InitChatPrompt(string prompt, string encoding = "UTF-8")
@@ -408,8 +428,8 @@ namespace LLama
         {
             var stateSize = NativeApi.llama_get_state_size(_ctx);
             byte[] stateMemory = new byte[stateSize];
-            int nbytes = (int)NativeApi.llama_copy_state_data(_ctx, stateMemory);
-            File.WriteAllBytes(filename, stateMemory.Take(nbytes).ToArray());
+            NativeApi.llama_copy_state_data(_ctx, stateMemory);
+            File.WriteAllBytes(filename, stateMemory);
         }
 
         /// <summary>
@@ -421,7 +441,8 @@ namespace LLama
         public void LoadState(string filename, bool clearPreviousEmbed = true)
         {
             var stateMemory = File.ReadAllBytes(filename);
-            if (stateMemory.Length != (int)NativeApi.llama_get_state_size(_ctx))
+            int stateSize = (int)NativeApi.llama_get_state_size(_ctx);
+            if (stateMemory.Length != stateSize)
             {
                 throw new RuntimeError("Failed to validate state size.");
             }
@@ -478,10 +499,18 @@ namespace LLama
         public IEnumerable<string> Call(string text, string encoding = "UTF-8")
         {
             _is_antiprompt = false;
-            if (_n_past > 0)
+            if(_n_past > 0)
             {
                 _is_interacting = false;
             }
+            if (_is_interacting)
+            {
+                if (_verbose)
+                {
+                    Logger.Default.Warn("In interacting when calling the model, automatically changed it.");
+                }
+                _is_interacting = false;
+            }
             ProcessTextBeforeInfer(text, encoding);
 
             while ((_n_remain != 0 || _params.interactive) && !_is_interacting)
@@ -503,14 +532,6 @@ namespace LLama
 
                         // stop saving session if we run out of context
                         _path_session = "";
-
-                        // Console.WriteLine("\n---\n");
-                        // Console.Write("resetting: '");
-                        // for (int i = 0; i < embed.Count; i++) {
-                        //     Console.Write(llama_token_to_str(ctx, embed[i]));
-                        // }
-                        // Console.WriteLine("'\n");
-                        // Console.WriteLine("\n---\n");
                     }
 
                     // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)

LLama/LLamaModelV1.cs

@@ -73,7 +73,6 @@ namespace LLama
 
             _params = NativeApi.llama_context_default_params();
             _params.n_ctx = n_ctx;
-            _params.n_parts = n_parts;
             _params.seed = seed;
             _params.f16_kv = f16_kv;
             _params.logits_all = logits_all;

LLama/LLamaParams.cs

@@ -9,7 +9,6 @@ namespace LLama
         public int seed; // RNG seed
         public int n_threads = Math.Max(Environment.ProcessorCount / 2, 1); // number of threads (-1 = autodetect)
         public int n_predict = -1; // new tokens to predict
-        public int n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
         public int n_ctx = 512; // context size
         public int n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)
         public int n_keep = 0; // number of tokens to keep from initial prompt
@@ -58,7 +57,7 @@ namespace LLama
         public bool verbose_prompt = false; // print prompt tokens before generation
 
         public LLamaParams(int seed = 0, int n_threads = -1, int n_predict = -1,
-            int n_parts = -1, int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1,
+            int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1,
             Dictionary<llama_token, float> logit_bias = null, int top_k = 40, float top_p = 0.95f,
             float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f,
             int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f,
@@ -78,7 +77,6 @@ namespace LLama
                 this.n_threads = n_threads;
             }
             this.n_predict = n_predict;
-            this.n_parts = n_parts;
             this.n_ctx = n_ctx;
             this.n_batch = n_batch;
             this.n_keep = n_keep;

LLama/Native/LLamaContextParams.cs

@@ -14,10 +14,6 @@ namespace LLama.Native
         /// </summary>
         public int n_ctx;
         /// <summary>
-        /// -1 for default
-        /// </summary>
-        public int n_parts;
-        /// <summary>
         /// number of layers to store in VRAM
         /// </summary>
         public int n_gpu_layers;

LLama/Native/LLamaFtype.cs

@@ -11,7 +11,7 @@ namespace LLama.Native
         LLAMA_FTYPE_MOSTLY_Q4_0 = 2,  // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_1 = 3,  // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
-        LLAMA_FTYPE_MOSTLY_Q4_2 = 5,  // except 1d tensors
+        // LLAMA_FTYPE_MOSTLY_Q4_2 = 5,  // support has been removed
         // LLAMA_FTYPE_MOSTLY_Q4_3 (6) support has been removed
         LLAMA_FTYPE_MOSTLY_Q8_0 = 7,  // except 1d tensors
         LLAMA_FTYPE_MOSTLY_Q5_0 = 8,  // except 1d tensors

LLama/Native/NativeApi.cs

@@ -24,6 +24,7 @@ namespace LLama.Native
                     "3. The backend is not compatible with your system cuda environment. Please check and fix it. If the environment is " +
                     "expected not to be changed, then consider build llama.cpp from source or submit an issue to LLamaSharp.");
             }
+            NativeApi.llama_init_backend();
         }
         private const string libraryName = "libllama";
 

LLama/Quantizer.cs

@@ -24,7 +24,6 @@ namespace LLama
                 throw new ArgumentException($"The type {Enum.GetName(typeof(LLamaFtype), ftype)} is not a valid type " +
                     $"to perform quantization.");
             }
-            NativeApi.llama_init_backend();
             return NativeApi.llama_model_quantize(srcFileName, dstFilename, ftype, nthread) == 0;
         }
 
@@ -44,12 +43,12 @@ namespace LLama
 
         private static bool ValidateFtype(string ftype)
         {
-            return new string[] { "q4_0", "q4_1", "q4_2", "q5_0", "q5_1", "q8_0" }.Contains(ftype);
+            return new string[] { "q4_0", "q4_1", "q5_0", "q5_1", "q8_0" }.Contains(ftype);
         }
 
         private static bool ValidateFtype(LLamaFtype ftype)
         {
-            return ftype is LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2
+            return ftype is LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1
                 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0;
         }
 
@@ -59,7 +58,6 @@ namespace LLama
             {
                 LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 => "q4_0",
                 LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 => "q4_1",
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2 => "q4_2",
                 LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 => "q5_0",
                 LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 => "q5_1",
                 LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0 => "q8_0",
@@ -74,7 +72,6 @@ namespace LLama
             {
                 "q4_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0,
                 "q4_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1,
-                "q4_2" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2,
                 "q5_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0,
                 "q5_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1,
                 "q8_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0,

LLama/Utils.cs

@@ -18,8 +18,7 @@ namespace LLama
             var lparams = NativeApi.llama_context_default_params();
 
             lparams.n_ctx = @params.n_ctx;
-            lparams.n_parts = @params.n_parts;
-            lparams.n_gpu_layers = @params.n_gpu_layers;
+            lparams.n_gpu_layers = 16;
             lparams.seed = @params.seed;
             lparams.f16_kv = @params.memory_f16;
             lparams.use_mmap = @params.use_mmap;