diff --git a/LLama.Examples/ChatSession.cs b/LLama.Examples/ChatSession.cs index 4f927104..2fa64652 100644 --- a/LLama.Examples/ChatSession.cs +++ b/LLama.Examples/ChatSession.cs @@ -25,7 +25,7 @@ namespace LLama.Examples Console.ForegroundColor = ConsoleColor.Green; var question = Console.ReadLine(); Console.ForegroundColor = ConsoleColor.White; - var outputs = _session.Chat(question); + var outputs = _session.Chat(question, encoding: "UTF-8"); foreach (var output in outputs) { Console.Write(output); diff --git a/LLama.Examples/ChatWithLLamaModel.cs b/LLama.Examples/ChatWithLLamaModel.cs index db38fe35..c9133814 100644 --- a/LLama.Examples/ChatWithLLamaModel.cs +++ b/LLama.Examples/ChatWithLLamaModel.cs @@ -12,7 +12,7 @@ namespace LLama.Examples public ChatWithLLamaModel(string modelPath, string promptFilePath, string[] antiprompt) { _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 512, interactive: true, antiprompt: antiprompt.ToList(), - repeat_penalty: 1.0f), echo_input: false).WithPromptFile(promptFilePath); + repeat_penalty: 1.0f)).WithPromptFile(promptFilePath); } public void Run() diff --git a/LLama.Examples/InstructMode.cs b/LLama.Examples/InstructMode.cs new file mode 100644 index 00000000..ebc04e36 --- /dev/null +++ b/LLama.Examples/InstructMode.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLama.Examples +{ + public class InstructMode + { + LLamaModel _model; + public InstructMode(string modelPath, string promptFile) + { + _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 2048, n_predict: -1, top_k: 10000, instruct: true, + repeat_penalty: 1.1f, n_batch: 256, temp: 0.2f)).WithPromptFile(promptFile); + } + + public void Run() + { + Console.WriteLine("\n### Instruction:\n >"); + while (true) + { + Console.ForegroundColor = ConsoleColor.Green; + var question = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + var outputs = _model.Call(question); + foreach (var output in outputs) + { + Console.Write(output); + } + } + } + } +} diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index c7338f64..41881f9d 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -16,6 +16,24 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index 19a4dd51..97b0e3f3 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -2,31 +2,87 @@ using LLama.Examples; using LLama.Types; -int choice = 0; +Console.WriteLine("================LLamaSharp Examples==================\n"); -if(choice == 0) -{ - ChatSession chat = new(@"", @"", new string[] { "User:" }); - chat.Run(); -} -else if(choice == 1) -{ - ChatWithLLamaModel chat = new(@"", "", new string[] { "User:" }); - chat.Run(); -} -else if(choice == 2) -{ - ChatWithLLamaModelV1 chat = new(@""); - chat.Run(); -} -else if (choice == 3) // quantization -{ - Quantize q = new Quantize(); - q.Run(@"", - @"", "q4_1"); -} -else if (choice == 4) // get the embeddings only +Console.WriteLine("Please input a number to choose an example to run:"); +Console.WriteLine("0: Run a chat session."); +Console.WriteLine("1: Run a LLamaModel to chat."); +Console.WriteLine("2: Quantize a model."); +Console.WriteLine("3: Get the embeddings of a message."); +Console.WriteLine("4: Run a LLamaModel with instruct mode."); +Console.WriteLine("5: Load and save state of LLamaModel."); + + +while (true) { - GetEmbeddings em = new GetEmbeddings(@""); - em.Run("Hello, what is python?"); + Console.Write("\nYour choice: "); + int choice = int.Parse(Console.ReadLine()); + + if (choice == 0) + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + ChatSession chat = new(modelPath, "Assets/chat-with-bob.txt", new string[] { "User:" }); + chat.Run(); + } + else if (choice == 1) + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + ChatWithLLamaModel chat = new(modelPath, "Assets/chat-with-bob.txt", new string[] { "User:" }); + chat.Run(); + } + else if (choice == 2) // quantization + { + Console.Write("Please input your original model path: "); + var inputPath = Console.ReadLine(); + Console.Write("Please input your output model path: "); + var outputPath = Console.ReadLine(); + Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): "); + var quantizeType = Console.ReadLine(); + Quantize q = new Quantize(); + q.Run(inputPath, outputPath, quantizeType); + } + else if (choice == 3) // get the embeddings only + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + GetEmbeddings em = new GetEmbeddings(modelPath); + Console.Write("Please input the text: "); + var text = Console.ReadLine(); + em.Run(text); + } + else if (choice == 4) // instruct mode + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + InstructMode im = new InstructMode(modelPath, "Assets/alpaca.txt"); + Console.WriteLine("Here's a simple example for using instruct mode. You can input some words and let AI " + + "complete it for you. For example: Write a story about a fox that wants to make friend with human. No less than 200 words."); + im.Run(); + } + else if (choice == 5) // load and save state + { + Console.Write("Please input your model path: "); + var modelPath = Console.ReadLine(); + Console.Write("Please input your state file path: "); + var statePath = Console.ReadLine(); + SaveAndLoadState sals = new(modelPath, File.ReadAllText(@"D:\development\llama\llama.cpp\prompts\alpaca.txt")); + sals.Run("Write a story about a fox that wants to make friend with human. No less than 200 words."); + sals.SaveState(statePath); + sals.Dispose(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + + // create a new model to load the state. + SaveAndLoadState sals2 = new(modelPath, ""); + sals2.LoadState(statePath); + sals2.Run("Tell me more things about the fox in the story you told me."); + } + else + { + Console.WriteLine("Cannot parse your choice. Please select again."); + continue; + } + break; } \ No newline at end of file diff --git a/LLama.Examples/SaveAndLoadState.cs b/LLama.Examples/SaveAndLoadState.cs new file mode 100644 index 00000000..dceac634 --- /dev/null +++ b/LLama.Examples/SaveAndLoadState.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLama.Examples +{ + public class SaveAndLoadState: IDisposable + { + LLamaModel _model; + public SaveAndLoadState(string modelPath, string prompt) + { + _model = new LLamaModel(new LLamaParams(model: modelPath, n_ctx: 2048, n_predict: -1, top_k: 10000, instruct: true, + repeat_penalty: 1.1f, n_batch: 256, temp: 0.2f)).WithPrompt(prompt); + } + + public void Run(string question) + { + // Only run once here. + Console.Write("\nUser:"); + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine(question); + Console.ForegroundColor = ConsoleColor.White; + var outputs = _model.Call(question); + foreach (var output in outputs) + { + Console.Write(output); + } + } + + public void SaveState(string filename) + { + _model.SaveState(filename); + Console.WriteLine("Saved state!"); + } + + public void LoadState(string filename) + { + _model.LoadState(filename); + Console.WriteLine("Loaded state!"); + } + + public void Dispose() + { + _model.Dispose(); + } + } +} diff --git a/LLama/LLamaModel.cs b/LLama/LLamaModel.cs index 068f0f4a..5e8a6bb6 100644 --- a/LLama/LLamaModel.cs +++ b/LLama/LLamaModel.cs @@ -25,6 +25,7 @@ namespace LLama bool _is_interacting; bool _is_antiprompt; bool _input_echo; + bool _verbose; // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session // if we loaded a session with at least 75% similarity. It's currently just used to speed up the @@ -37,6 +38,17 @@ namespace LLama List _embed; public string Name { get; set; } + public bool Verbose + { + get + { + return _verbose; + } + set + { + _verbose = value; + } + } public SafeLLamaContextHandle NativeHandle => _ctx; /// @@ -44,7 +56,6 @@ namespace LLama /// /// The model file path. /// The model name. - /// Whether to print the input messages. /// Whether to print details when running the model. /// /// @@ -88,7 +99,7 @@ namespace LLama /// /// /// - public LLamaModel(string model_path, string model_name, bool echo_input = false, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, + public LLamaModel(string model_path, string model_name, bool verbose = false, int seed = 0, int n_threads = -1, int n_predict = -1, int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1, Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, @@ -141,15 +152,24 @@ namespace LLama use_mlock: use_mlock, mem_test: mem_test, verbose_prompt: verbose_prompt), - model_name, echo_input, verbose, encoding) + model_name, verbose, encoding) { } - public unsafe LLamaModel(LLamaParams @params, string name = "", bool echo_input = false, bool verbose = false, string encoding = "UTF-8") + /// + /// + /// + /// The LLamaModel params + /// Model name + /// Whether to output the detailed info. + /// + /// + public unsafe LLamaModel(LLamaParams @params, string name = "", bool verbose = false, string encoding = "UTF-8") { Name = name; _params = @params; + _verbose = verbose; _ctx = Utils.llama_init_from_gpt_params(ref _params); // Add a space in front of the first character to match OG llama tokenizer behavior @@ -197,7 +217,7 @@ namespace LLama } // enable interactive mode if reverse prompt or interactive start is specified - if (_params.antiprompt.Count != 0 || _params.interactive_first) + if (_params.interactive_first) { _params.interactive = true; } @@ -233,10 +253,10 @@ namespace LLama if (verbose) { Logger.Default.Info($"sampling: repeat_last_n = {_params.repeat_last_n}, " + - $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + - $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + - $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + - $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); + $"repeat_penalty = {_params.repeat_penalty}, presence_penalty = {_params.presence_penalty}, " + + $"frequency_penalty = {_params.frequency_penalty}, top_k = {_params.top_k}, tfs_z = {_params.tfs_z}," + + $" top_p = {_params.top_p}, typical_p = {_params.typical_p}, temp = {_params.temp}, mirostat = {_params.mirostat}," + + $" mirostat_lr = {_params.mirostat_eta}, mirostat_ent = {_params.mirostat_tau}"); Logger.Default.Info($"generate: n_ctx = {_n_ctx}, n_batch = {_params.n_batch}, n_predict = {_params.n_predict}, " + $"n_keep = {_params.n_keep}"); Logger.Default.Info("\n"); @@ -254,7 +274,7 @@ namespace LLama } _is_antiprompt = false; - _input_echo = echo_input; + _input_echo = false; _n_past = 0; _n_remain = _params.n_predict; _n_consumed = 0; @@ -315,6 +335,7 @@ namespace LLama throw new ArgumentException($"prompt is too long ({_embed_inp.Count} tokens, max {_n_ctx - 4})"); } _need_to_save_session = !string.IsNullOrEmpty(_path_session) && n_matching_session_tokens < (ulong)(_embed_inp.Count * 3 / 4); + return this; } @@ -328,23 +349,23 @@ namespace LLama return WithPrompt(File.ReadAllText(promptFileName)); } - private string ProcessTextBeforeInfer(string text, string encoding) + private void ProcessTextBeforeInfer(string text, string encoding) { if (!string.IsNullOrEmpty(_params.input_prefix)) { text = _params.input_prefix + text; } - if (!text.EndsWith("\n")) - { - text += "\n"; - } + //if (!text.EndsWith("\n")) + //{ + // text += "\n"; + //} if (text.Length > 1) { // append input suffix if any if (!string.IsNullOrEmpty(_params.input_suffix)) { text += _params.input_suffix; - Console.Write(_params.input_suffix); + //yield return _params.input_suffix; } // instruct mode: insert instruction prefix @@ -365,7 +386,6 @@ namespace LLama _n_remain -= line_inp.Count; } - return text; } public void InitChatPrompt(string prompt, string encoding = "UTF-8") @@ -408,8 +428,8 @@ namespace LLama { var stateSize = NativeApi.llama_get_state_size(_ctx); byte[] stateMemory = new byte[stateSize]; - int nbytes = (int)NativeApi.llama_copy_state_data(_ctx, stateMemory); - File.WriteAllBytes(filename, stateMemory.Take(nbytes).ToArray()); + NativeApi.llama_copy_state_data(_ctx, stateMemory); + File.WriteAllBytes(filename, stateMemory); } /// @@ -421,7 +441,8 @@ namespace LLama public void LoadState(string filename, bool clearPreviousEmbed = true) { var stateMemory = File.ReadAllBytes(filename); - if (stateMemory.Length != (int)NativeApi.llama_get_state_size(_ctx)) + int stateSize = (int)NativeApi.llama_get_state_size(_ctx); + if (stateMemory.Length != stateSize) { throw new RuntimeError("Failed to validate state size."); } @@ -478,10 +499,18 @@ namespace LLama public IEnumerable Call(string text, string encoding = "UTF-8") { _is_antiprompt = false; - if (_n_past > 0) + if(_n_past > 0) { _is_interacting = false; } + if (_is_interacting) + { + if (_verbose) + { + Logger.Default.Warn("In interacting when calling the model, automatically changed it."); + } + _is_interacting = false; + } ProcessTextBeforeInfer(text, encoding); while ((_n_remain != 0 || _params.interactive) && !_is_interacting) @@ -503,14 +532,6 @@ namespace LLama // stop saving session if we run out of context _path_session = ""; - - // Console.WriteLine("\n---\n"); - // Console.Write("resetting: '"); - // for (int i = 0; i < embed.Count; i++) { - // Console.Write(llama_token_to_str(ctx, embed[i])); - // } - // Console.WriteLine("'\n"); - // Console.WriteLine("\n---\n"); } // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) diff --git a/LLama/LLamaModelV1.cs b/LLama/LLamaModelV1.cs index c984a63d..b9e5b822 100644 --- a/LLama/LLamaModelV1.cs +++ b/LLama/LLamaModelV1.cs @@ -73,7 +73,6 @@ namespace LLama _params = NativeApi.llama_context_default_params(); _params.n_ctx = n_ctx; - _params.n_parts = n_parts; _params.seed = seed; _params.f16_kv = f16_kv; _params.logits_all = logits_all; diff --git a/LLama/LLamaParams.cs b/LLama/LLamaParams.cs index b08ada36..4380bcfd 100644 --- a/LLama/LLamaParams.cs +++ b/LLama/LLamaParams.cs @@ -9,7 +9,6 @@ namespace LLama public int seed; // RNG seed public int n_threads = Math.Max(Environment.ProcessorCount / 2, 1); // number of threads (-1 = autodetect) public int n_predict = -1; // new tokens to predict - public int n_parts = -1; // amount of model parts (-1 = determine from model dimensions) public int n_ctx = 512; // context size public int n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) public int n_keep = 0; // number of tokens to keep from initial prompt @@ -58,7 +57,7 @@ namespace LLama public bool verbose_prompt = false; // print prompt tokens before generation public LLamaParams(int seed = 0, int n_threads = -1, int n_predict = -1, - int n_parts = -1, int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1, + int n_ctx = 512, int n_batch = 512, int n_keep = 0, int n_gpu_layers = -1, Dictionary logit_bias = null, int top_k = 40, float top_p = 0.95f, float tfs_z = 1.00f, float typical_p = 1.00f, float temp = 0.80f, float repeat_penalty = 1.10f, int repeat_last_n = 64, float frequency_penalty = 0.00f, float presence_penalty = 0.00f, @@ -78,7 +77,6 @@ namespace LLama this.n_threads = n_threads; } this.n_predict = n_predict; - this.n_parts = n_parts; this.n_ctx = n_ctx; this.n_batch = n_batch; this.n_keep = n_keep; diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs index 0fd0a51a..aa448ab9 100644 --- a/LLama/Native/LLamaContextParams.cs +++ b/LLama/Native/LLamaContextParams.cs @@ -14,10 +14,6 @@ namespace LLama.Native /// public int n_ctx; /// - /// -1 for default - /// - public int n_parts; - /// /// number of layers to store in VRAM /// public int n_gpu_layers; diff --git a/LLama/Native/LLamaFtype.cs b/LLama/Native/LLamaFtype.cs index 84a4e372..8ecc224f 100644 --- a/LLama/Native/LLamaFtype.cs +++ b/LLama/Native/LLamaFtype.cs @@ -11,7 +11,7 @@ namespace LLama.Native LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 - LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed // LLAMA_FTYPE_MOSTLY_Q4_3 (6) support has been removed LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index e43ed2f2..71bfcc6b 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -24,6 +24,7 @@ namespace LLama.Native "3. The backend is not compatible with your system cuda environment. Please check and fix it. If the environment is " + "expected not to be changed, then consider build llama.cpp from source or submit an issue to LLamaSharp."); } + NativeApi.llama_init_backend(); } private const string libraryName = "libllama"; diff --git a/LLama/Quantizer.cs b/LLama/Quantizer.cs index 05a9fe41..9f8b22df 100644 --- a/LLama/Quantizer.cs +++ b/LLama/Quantizer.cs @@ -24,7 +24,6 @@ namespace LLama throw new ArgumentException($"The type {Enum.GetName(typeof(LLamaFtype), ftype)} is not a valid type " + $"to perform quantization."); } - NativeApi.llama_init_backend(); return NativeApi.llama_model_quantize(srcFileName, dstFilename, ftype, nthread) == 0; } @@ -44,12 +43,12 @@ namespace LLama private static bool ValidateFtype(string ftype) { - return new string[] { "q4_0", "q4_1", "q4_2", "q5_0", "q5_1", "q8_0" }.Contains(ftype); + return new string[] { "q4_0", "q4_1", "q5_0", "q5_1", "q8_0" }.Contains(ftype); } private static bool ValidateFtype(LLamaFtype ftype) { - return ftype is LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2 + return ftype is LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0; } @@ -59,7 +58,6 @@ namespace LLama { LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 => "q4_0", LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 => "q4_1", - LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2 => "q4_2", LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 => "q5_0", LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 => "q5_1", LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0 => "q8_0", @@ -74,7 +72,6 @@ namespace LLama { "q4_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0, "q4_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1, - "q4_2" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_2, "q5_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0, "q5_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1, "q8_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0, diff --git a/LLama/Utils.cs b/LLama/Utils.cs index 25b8a4dc..a915859d 100644 --- a/LLama/Utils.cs +++ b/LLama/Utils.cs @@ -18,8 +18,7 @@ namespace LLama var lparams = NativeApi.llama_context_default_params(); lparams.n_ctx = @params.n_ctx; - lparams.n_parts = @params.n_parts; - lparams.n_gpu_layers = @params.n_gpu_layers; + lparams.n_gpu_layers = 16; lparams.seed = @params.seed; lparams.f16_kv = @params.memory_f16; lparams.use_mmap = @params.use_mmap;