diff --git a/LLama.Examples/NewVersion/InstructModeExecute.cs b/LLama.Examples/NewVersion/InstructModeExecute.cs index 59fcf09b..303c8644 100644 --- a/LLama.Examples/NewVersion/InstructModeExecute.cs +++ b/LLama.Examples/NewVersion/InstructModeExecute.cs @@ -22,7 +22,7 @@ namespace LLama.Examples.NewVersion "make friend with human, no less than 200 words.\""); Console.ForegroundColor = ConsoleColor.White; - var inferenceParams = new InferenceParams() { Temperature = 0.8f, MaxTokens = 300 }; + var inferenceParams = new InferenceParams() { Temperature = 0.8f, MaxTokens = 600 }; while (true) { diff --git a/LLama.Examples/NewVersion/InteractiveModeExecute.cs b/LLama.Examples/NewVersion/InteractiveModeExecute.cs index ba29baac..23afcadf 100644 --- a/LLama.Examples/NewVersion/InteractiveModeExecute.cs +++ b/LLama.Examples/NewVersion/InteractiveModeExecute.cs @@ -18,12 +18,12 @@ namespace LLama.Examples.NewVersion InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 256))); Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 64 and the context size is 256. (an example for small scale usage)"); + Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 128 and the context size is 256. (an example for small scale usage)"); Console.ForegroundColor = ConsoleColor.White; Console.Write(prompt); - var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" }, MaxTokens = 64 }; + var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List { "User:" }, MaxTokens = 128 }; while (true) { diff --git a/LLama.Examples/NewVersion/LoadAndSaveSession.cs b/LLama.Examples/NewVersion/LoadAndSaveSession.cs index ae43d53b..722ec3e0 100644 --- a/LLama.Examples/NewVersion/LoadAndSaveSession.cs +++ b/LLama.Examples/NewVersion/LoadAndSaveSession.cs @@ -47,7 +47,7 @@ namespace LLama.Examples.NewVersion ex.Model.Dispose(); ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); - session = new ChatSession(ex).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); + session = new ChatSession(ex); session.LoadSession(statePath); Console.ForegroundColor = ConsoleColor.Yellow; diff --git a/LLama/Common/InferenceParams.cs b/LLama/Common/InferenceParams.cs index 11b4c355..5935a0ee 100644 --- a/LLama/Common/InferenceParams.cs +++ b/LLama/Common/InferenceParams.cs @@ -5,6 +5,9 @@ using System.Text; namespace LLama.Common { using llama_token = Int32; + /// + /// The paramters used for inference. + /// public class InferenceParams { /// diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index 46ebd5c3..2a591bcd 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -4,13 +4,23 @@ using System.Text; namespace LLama.Common { + /// + /// The parameters for initializing a LLama model. + /// public class ModelParams { /// /// Model context size (n_ctx) /// public int ContextSize { get; set; } = 512; - + /// + /// the GPU that is used for scratch and small tensors + /// + public int MainGpu { get; set; } = 0; + /// + /// if true, reduce VRAM usage at the cost of performance + /// + public bool LowVram { get; set; } = false; /// /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// @@ -40,6 +50,10 @@ namespace LLama.Common /// public string ModelPath { get; set; } /// + /// model alias + /// + public string ModelAlias { get; set; } = "unknown"; + /// /// lora adapter path (lora_adapter) /// public string LoraAdapter { get; set; } = string.Empty; @@ -67,6 +81,11 @@ namespace LLama.Common /// public bool EmbeddingMode { get; set; } = false; + /// + /// how split tensors should be distributed across GPUs + /// + public float[] TensorSplits { get; set; } = new float[] { 0 }; + /// /// /// diff --git a/LLama/LLamaInstructExecutor.cs b/LLama/LLamaInstructExecutor.cs index a2bc9c74..613a5f46 100644 --- a/LLama/LLamaInstructExecutor.cs +++ b/LLama/LLamaInstructExecutor.cs @@ -17,6 +17,7 @@ namespace LLama public class InstructExecutor : StatefulExecutorBase { bool _is_prompt_run = true; + string _instructionPrefix; llama_token[] _inp_pfx; llama_token[] _inp_sfx; /// @@ -30,6 +31,7 @@ namespace LLama { _inp_pfx = _model.Tokenize(instructionPrefix, true).ToArray(); _inp_sfx = _model.Tokenize(instructionSuffix, false).ToArray(); + _instructionPrefix = instructionPrefix; } /// @@ -104,6 +106,11 @@ namespace LLama /// protected override void PreprocessInputs(string text, InferStateArgs args) { + if(args.Antiprompts is null) + { + args.Antiprompts = new List(); + } + args.Antiprompts.Add(_instructionPrefix); if (_is_prompt_run) { // When running the first input (prompt) in inteactive mode, we should specially process it. diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 7088b70e..59468d5b 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -8,7 +8,7 @@ AnyCPU;x64;Arm64 True - 0.4.0 + 0.4.1 Yaohui Liu, Haiping Chen SciSharp STACK true @@ -21,7 +21,7 @@ The .NET binding of LLama.cpp, providing APIs to run the model and deploy it on Web. For model weights to run, please go to https://github.com/SciSharp/LLamaSharp for more information. - LLamaSharp 0.4.0 supports better APIs than v0.3.0. Note that many break changes were made in this version. APIs of v0.3.0 were moved to LLama.Old namespace. + LLamaSharp 0.4.1 followed up the master branch of llama.cpp. (commit id: aacdbd4) MIT packages diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 4540a123..ccafdf1b 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -44,6 +44,9 @@ namespace LLama.Native [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] public static extern bool llama_mlock_supported(); + [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern int llama_eval_export(SafeLLamaContextHandle ctx, string fname); + /// /// Various functions for loading a ggml llama model. /// Allocate (almost) all memory needed for the model. diff --git a/LLama/Utils.cs b/LLama/Utils.cs index 52aa0aab..b6f1b7b4 100644 --- a/LLama/Utils.cs +++ b/LLama/Utils.cs @@ -18,6 +18,8 @@ namespace LLama var lparams = NativeApi.llama_context_default_params(); lparams.n_ctx = @params.ContextSize; + lparams.n_batch = @params.BatchSize; + lparams.main_gpu = @params.MainGpu; lparams.n_gpu_layers = @params.GpuLayerCount; lparams.seed = @params.Seed; lparams.f16_kv = @params.UseFp16Memory; @@ -25,6 +27,17 @@ namespace LLama lparams.use_mlock = @params.UseMemoryLock; lparams.logits_all = @params.Perplexity; lparams.embedding = @params.EmbeddingMode; + lparams.low_vram = @params.LowVram; + + if(@params.TensorSplits.Length != 1) + { + throw new ArgumentException("Currently multi-gpu support is not supported by " + + "both llama.cpp and LLamaSharp."); + } + lparams.tensor_split = new TensorSplits() + { + Item1 = @params.TensorSplits[0] + }; if (!File.Exists(@params.ModelPath)) { diff --git a/LLama/runtimes/libllama-cuda11.dll b/LLama/runtimes/libllama-cuda11.dll index 4b416279..c7936db2 100644 Binary files a/LLama/runtimes/libllama-cuda11.dll and b/LLama/runtimes/libllama-cuda11.dll differ diff --git a/LLama/runtimes/libllama-cuda11.so b/LLama/runtimes/libllama-cuda11.so index d82bb29d..508b4e1b 100644 Binary files a/LLama/runtimes/libllama-cuda11.so and b/LLama/runtimes/libllama-cuda11.so differ diff --git a/LLama/runtimes/libllama-cuda12.dll b/LLama/runtimes/libllama-cuda12.dll index f7ec46a4..97c1a9b1 100644 Binary files a/LLama/runtimes/libllama-cuda12.dll and b/LLama/runtimes/libllama-cuda12.dll differ diff --git a/LLama/runtimes/libllama-cuda12.so b/LLama/runtimes/libllama-cuda12.so index 962323df..bcb7d3ad 100644 Binary files a/LLama/runtimes/libllama-cuda12.so and b/LLama/runtimes/libllama-cuda12.so differ diff --git a/LLama/runtimes/libllama.dll b/LLama/runtimes/libllama.dll index 728bb24c..ed055e88 100644 Binary files a/LLama/runtimes/libllama.dll and b/LLama/runtimes/libllama.dll differ diff --git a/LLama/runtimes/libllama.dylib b/LLama/runtimes/libllama.dylib deleted file mode 100755 index ea9746d2..00000000 Binary files a/LLama/runtimes/libllama.dylib and /dev/null differ diff --git a/LLama/runtimes/libllama.so b/LLama/runtimes/libllama.so index d0415604..5acc506a 100644 Binary files a/LLama/runtimes/libllama.so and b/LLama/runtimes/libllama.so differ