diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index 3ecacdfe..dcd40c19 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -30,6 +30,7 @@ + diff --git a/LLama.Examples/NewVersion/GetEmbeddings.cs b/LLama.Examples/NewVersion/GetEmbeddings.cs index fe9e3ea8..1e5b19be 100644 --- a/LLama.Examples/NewVersion/GetEmbeddings.cs +++ b/LLama.Examples/NewVersion/GetEmbeddings.cs @@ -4,7 +4,7 @@ namespace LLama.Examples.NewVersion { public class GetEmbeddings { - public static void Run() + public static Task Run() { Console.Write("Please input your model path: "); var modelPath = Console.ReadLine(); @@ -23,6 +23,7 @@ namespace LLama.Examples.NewVersion Console.WriteLine(string.Join(", ", embedder.GetEmbeddings(text))); Console.WriteLine(); } + return Task.CompletedTask; } } } diff --git a/LLama.Examples/NewVersion/QuantizeModel.cs b/LLama.Examples/NewVersion/QuantizeModel.cs index 71966af8..456d8929 100644 --- a/LLama.Examples/NewVersion/QuantizeModel.cs +++ b/LLama.Examples/NewVersion/QuantizeModel.cs @@ -2,7 +2,7 @@ { public class QuantizeModel { - public static void Run() + public static Task Run() { Console.Write("Please input your original model path: "); var inputPath = Console.ReadLine(); @@ -21,6 +21,8 @@ { Console.WriteLine("Quantization failed!"); } + + return Task.CompletedTask; } } } diff --git a/LLama.Examples/NewVersion/TestRunner.cs b/LLama.Examples/NewVersion/TestRunner.cs index a21a2eed..c89cba30 100644 --- a/LLama.Examples/NewVersion/TestRunner.cs +++ b/LLama.Examples/NewVersion/TestRunner.cs @@ -1,109 +1,54 @@ -namespace LLama.Examples.NewVersion +using System.Linq.Expressions; +using Spectre.Console; + +namespace LLama.Examples.NewVersion { public class NewVersionTestRunner { + static Dictionary> Examples = new Dictionary> + { + {"Run a chat session without stripping the role names.", () => ChatSessionWithRoleName.Run()}, + {"Run a chat session with the role names stripped.",()=> ChatSessionStripRoleName.Run()}, + {"Interactive mode chat by using executor.",()=> InteractiveModeExecute.Run()}, + {"Instruct mode chat by using executor.",()=> InstructModeExecute.Run()}, + {"Stateless mode chat by using executor.",()=> StatelessModeExecute.Run()}, + {"Load and save chat session.",()=> SaveAndLoadSession.Run()}, + {"Load and save state of model and executor.",()=> LoadAndSaveState.Run()}, + {"Get embeddings from LLama model.",()=> GetEmbeddings.Run()}, + {"Quantize the model.",()=> QuantizeModel.Run()}, + {"Automatic conversation.",()=> TalkToYourself.Run()}, + {"Constrain response to json format using grammar.",()=> GrammarJsonResponse.Run()}, + {"Semantic Kernel Prompt.",()=> SemanticKernelPrompt.Run()}, + {"Semantic Kernel Chat.",()=> SemanticKernelChat.Run()}, + {"Semantic Kernel Memory.",()=> SemanticKernelMemory.Run()}, + {"Coding Assistant.",()=> CodingAssistant.Run()}, + {"Batch Decoding.",()=> BatchedDecoding.Run()}, + {"SK Kernel Memory.",()=> KernelMemory.Run()}, + {"Exit", ()=> Task.CompletedTask} + }; public static async Task Run() { - Console.WriteLine("================LLamaSharp Examples (New Version)==================\n"); - - Console.WriteLine("Please input a number to choose an example to run:"); - Console.WriteLine("0: Run a chat session without stripping the role names."); - Console.WriteLine("1: Run a chat session with the role names stripped."); - Console.WriteLine("2: Interactive mode chat by using executor."); - Console.WriteLine("3: Instruct mode chat by using executor."); - Console.WriteLine("4: Stateless mode chat by using executor."); - Console.WriteLine("5: Load and save chat session."); - Console.WriteLine("6: Load and save state of model and executor."); - Console.WriteLine("7: Get embeddings from LLama model."); - Console.WriteLine("8: Quantize the model."); - Console.WriteLine("9: Automatic conversation."); - Console.WriteLine("10: Constrain response to json format using grammar."); - Console.WriteLine("11: Semantic Kernel Prompt."); - Console.WriteLine("12: Semantic Kernel Chat."); - Console.WriteLine("13: Semantic Kernel Memory."); - Console.WriteLine("14: Coding Assistant."); - Console.WriteLine("15: Batch Decoding."); - Console.WriteLine("16: SK Kernel Memory."); + AnsiConsole.Write(new Rule("LLamaSharp Examples")); while (true) { - Console.Write("\nYour choice: "); - int choice = int.Parse(Console.ReadLine()); + var choice = AnsiConsole.Prompt( + new SelectionPrompt() + .Title("Please choose[green] an example[/] to run: ") + .AddChoices(Examples.Keys)); - if (choice == 0) - { - await ChatSessionWithRoleName.Run(); - } - else if (choice == 1) - { - await ChatSessionStripRoleName.Run(); - } - else if (choice == 2) - { - await InteractiveModeExecute.Run(); - } - else if (choice == 3) - { - await InstructModeExecute.Run(); - } - else if (choice == 4) - { - await StatelessModeExecute.Run(); - } - else if (choice == 5) - { - await SaveAndLoadSession.Run(); - } - else if (choice == 6) - { - await LoadAndSaveState.Run(); - } - else if (choice == 7) - { - GetEmbeddings.Run(); - } - else if (choice == 8) - { - QuantizeModel.Run(); - } - else if (choice == 9) - { - await TalkToYourself.Run(); - } - else if (choice == 10) - { - await GrammarJsonResponse.Run(); - } - else if (choice == 11) - { - await SemanticKernelPrompt.Run(); - } - else if (choice == 12) - { - await SemanticKernelChat.Run(); - } - else if (choice == 13) - { - await SemanticKernelMemory.Run(); - } - else if (choice == 14) - { - await CodingAssistant.Run(); - } - else if (choice == 15) - { - await BatchedDecoding.Run(); - } - else if (choice == 16) - { - await KernelMemory.Run(); - } - else + + if (Examples.TryGetValue(choice, out var example)) { - Console.WriteLine("Cannot parse your choice. Please select again."); - continue; + if (choice == "Exit") + { + break; + } + AnsiConsole.Write(new Rule(choice)); + await example(); } - break; + + AnsiConsole.Clear(); } } } diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index 6a63ccc3..8cbf2f09 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -17,9 +17,9 @@ namespace LLama.Web.Common public int MaxInstances { get; set; } /// - /// Model context size (n_ctx) + /// Model context size (n_ctx). Null to use value from model. /// - public uint ContextSize { get; set; } = 512; + public uint? ContextSize { get; set; } /// /// the GPU that is used for scratch and small tensors diff --git a/LLama/Abstractions/IContextParams.cs b/LLama/Abstractions/IContextParams.cs index 8ff6d7cc..a2ac24f1 100644 --- a/LLama/Abstractions/IContextParams.cs +++ b/LLama/Abstractions/IContextParams.cs @@ -8,9 +8,9 @@ namespace LLama.Abstractions; public interface IContextParams { /// - /// Model context size (n_ctx) + /// Model context size (n_ctx). Null to use value from model file. /// - uint ContextSize { get; set; } + uint? ContextSize { get; set; } /// /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) diff --git a/LLama/Common/FixedSizeQueue.cs b/LLama/Common/FixedSizeQueue.cs index d4577a47..37fb1cf5 100644 --- a/LLama/Common/FixedSizeQueue.cs +++ b/LLama/Common/FixedSizeQueue.cs @@ -43,7 +43,7 @@ namespace LLama.Common /// public FixedSizeQueue(int size, IEnumerable data) { -#if !NETSTANDARD2_0 +#if NET6_0_OR_GREATER // Try to check the size without enumerating the entire IEnumerable. This may not be able to get the count, // in which case we'll have to check later if (data.TryGetNonEnumeratedCount(out var dataCount) && dataCount > size) diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index ee5bd3e4..9561e482 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -12,105 +12,68 @@ namespace LLama.Common public record ModelParams : ILLamaParams { - /// - /// Model context size (n_ctx) - /// - public uint ContextSize { get; set; } = 512; - /// - /// the GPU that is used for scratch and small tensors - /// + /// + public uint? ContextSize { get; set; } + + /// public int MainGpu { get; set; } = 0; - /// - /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) - /// + /// public int GpuLayerCount { get; set; } = 20; - /// - /// Seed for the random number generator (seed) - /// + + /// public uint Seed { get; set; } = 0xFFFFFFFF; - /// - /// Use f16 instead of f32 for memory kv (memory_f16) - /// + + /// public bool UseFp16Memory { get; set; } = true; - /// - /// Use mmap for faster loads (use_mmap) - /// + + /// public bool UseMemorymap { get; set; } = true; - /// - /// Use mlock to keep model in memory (use_mlock) - /// + + /// public bool UseMemoryLock { get; set; } - /// - /// Compute perplexity over the prompt (perplexity) - /// + + /// public bool Perplexity { get; set; } - /// - /// Model path (model) - /// + + /// public string ModelPath { get; set; } - /// - /// List of LoRAs to apply - /// + /// public AdapterCollection LoraAdapters { get; set; } = new(); - /// - /// base model path for the lora adapter (lora_base) - /// + /// public string LoraBase { get; set; } = string.Empty; - /// - /// Number of threads (null = autodetect) (n_threads) - /// + /// public uint? Threads { get; set; } - /// - /// Number of threads to use for batch processing (null = autodetect) (n_threads) - /// + /// public uint? BatchThreads { get; set; } - /// - /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) - /// + /// public uint BatchSize { get; set; } = 512; - /// - /// Whether to use embedding mode. (embedding) Note that if this is set to true, - /// The LLamaModel won't produce text response anymore. - /// + /// public bool EmbeddingMode { get; set; } - /// - /// how split tensors should be distributed across GPUs. - /// - /// "[ 3, 2 ]" will assign 60% of the data to GPU 0 and 40% to GPU 1. + /// [JsonConverter(typeof(TensorSplitsCollectionConverter))] public TensorSplitsCollection TensorSplits { get; set; } = new(); - /// - /// RoPE base frequency - /// - public float? RopeFrequencyBase { get; set; } + /// + public float? RopeFrequencyBase { get; set; } - /// - /// RoPE frequency scaling factor - /// - public float? RopeFrequencyScale { get; set; } + /// + public float? RopeFrequencyScale { get; set; } - /// - /// Use experimental mul_mat_q kernels - /// - public bool MulMatQ { get; set; } + /// + public bool MulMatQ { get; set; } - /// - /// Load vocab only (no weights) - /// + /// public bool VocabOnly { get; set; } - /// - /// The encoding to use to convert text for the model - /// + /// [JsonConverter(typeof(EncodingConverter))] public Encoding Encoding { get; set; } = Encoding.UTF8; diff --git a/LLama/Extensions/DictionaryExtensions.cs b/LLama/Extensions/DictionaryExtensions.cs index a39ed7e8..1af0e9e1 100644 --- a/LLama/Extensions/DictionaryExtensions.cs +++ b/LLama/Extensions/DictionaryExtensions.cs @@ -9,6 +9,8 @@ namespace LLama.Extensions { return GetValueOrDefaultImpl(dictionary, key, defaultValue); } +#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER +#error Target framework not supported! #endif internal static TValue GetValueOrDefaultImpl(IReadOnlyDictionary dictionary, TKey key, TValue defaultValue) diff --git a/LLama/Extensions/EncodingExtensions.cs b/LLama/Extensions/EncodingExtensions.cs index e88d83a7..5005b16c 100644 --- a/LLama/Extensions/EncodingExtensions.cs +++ b/LLama/Extensions/EncodingExtensions.cs @@ -15,6 +15,8 @@ internal static class EncodingExtensions { return GetCharCountImpl(encoding, bytes); } +#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER +#error Target framework not supported! #endif internal static int GetCharsImpl(Encoding encoding, ReadOnlySpan bytes, Span output) diff --git a/LLama/Extensions/IContextParamsExtensions.cs b/LLama/Extensions/IContextParamsExtensions.cs index fcc9d372..ed59c9df 100644 --- a/LLama/Extensions/IContextParamsExtensions.cs +++ b/LLama/Extensions/IContextParamsExtensions.cs @@ -21,7 +21,7 @@ namespace LLama.Extensions public static void ToLlamaContextParams(this IContextParams @params, out LLamaContextParams result) { result = NativeApi.llama_context_default_params(); - result.n_ctx = @params.ContextSize; + result.n_ctx = @params.ContextSize ?? 0; result.n_batch = @params.BatchSize; result.seed = @params.Seed; result.f16_kv = @params.UseFp16Memory; diff --git a/LLama/Extensions/IEnumerableExtensions.cs b/LLama/Extensions/IEnumerableExtensions.cs index 9e01feb8..17428d29 100644 --- a/LLama/Extensions/IEnumerableExtensions.cs +++ b/LLama/Extensions/IEnumerableExtensions.cs @@ -10,6 +10,8 @@ namespace LLama.Extensions { return TakeLastImpl(source, count); } +#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER +#error Target framework not supported! #endif internal static IEnumerable TakeLastImpl(IEnumerable source, int count) diff --git a/LLama/Extensions/KeyValuePairExtensions.cs b/LLama/Extensions/KeyValuePairExtensions.cs index 6e12654d..233195ed 100644 --- a/LLama/Extensions/KeyValuePairExtensions.cs +++ b/LLama/Extensions/KeyValuePairExtensions.cs @@ -19,5 +19,7 @@ internal static class KeyValuePairExtensions first = pair.Key; second = pair.Value; } +#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER +#error Target framework not supported! #endif } \ No newline at end of file diff --git a/LLama/Extensions/ListExtensions.cs b/LLama/Extensions/ListExtensions.cs index 11a1d4f0..eb30a07a 100644 --- a/LLama/Extensions/ListExtensions.cs +++ b/LLama/Extensions/ListExtensions.cs @@ -5,7 +5,7 @@ namespace LLama.Extensions { internal static class ListExtensions { -#if NETSTANDARD2_0 +#if !NET6_0_OR_GREATER public static void EnsureCapacity(this List list, int capacity) { if (list.Capacity < capacity) diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index d525202f..4577515b 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -47,6 +47,7 @@ + diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs index 0a397a3d..9a0b2a8e 100644 --- a/LLama/Native/LLamaContextParams.cs +++ b/LLama/Native/LLamaContextParams.cs @@ -22,7 +22,7 @@ namespace LLama.Native public uint seed; /// - /// text context + /// text context, 0 = from model /// public uint n_ctx; diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 119a36fb..9998020e 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -1,8 +1,13 @@ using System; using System.Buffers; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using LLama.Exceptions; +using ManagedCuda; +#if NET6_0_OR_GREATER +using System.Runtime.Intrinsics.X86; +#endif #pragma warning disable IDE1006 // Naming Styles @@ -24,8 +29,9 @@ namespace LLama.Native { static NativeApi() { - // Try to load a preferred library, based on CPU feature detection - TryLoadLibrary(); +#if NET6_0_OR_GREATER + NativeLibrary.SetDllImportResolver(typeof(NativeApi).Assembly, LLamaImportResolver); +#endif try { @@ -44,63 +50,120 @@ namespace LLama.Native } /// - /// Try to load libllama, using CPU feature detection to try and load a more specialised DLL if possible + /// Get the cuda version if possible. /// - /// The library handle to unload later, or IntPtr.Zero if no library was loaded - private static IntPtr TryLoadLibrary() + /// -1 for no cuda + private static int GetCudaVersion() { -#if NET6_0_OR_GREATER - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + int deviceCount = CudaContext.GetDeviceCount(); + for (int deviceIndex = 0; deviceIndex < deviceCount; deviceIndex++) { - // All of the Windows libraries, in order of preference - return TryLoad("cu12.1.0/libllama.dll") - ?? TryLoad("cu11.7.1/libllama.dll") -#if NET8_0_OR_GREATER - ?? TryLoad("avx512/libllama.dll", System.Runtime.Intrinsics.X86.Avx512.IsSupported) -#endif - ?? TryLoad("avx2/libllama.dll", System.Runtime.Intrinsics.X86.Avx2.IsSupported) - ?? TryLoad("avx/libllama.dll", System.Runtime.Intrinsics.X86.Avx.IsSupported) - ?? IntPtr.Zero; + using (CudaContext ctx = new CudaContext(deviceIndex)) + { + var version = ctx.GetAPIVersionOfCurrentContext(); + return version.Major; + } } + return -1; + } - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - // All of the Linux libraries, in order of preference - return TryLoad("cu12.1.0/libllama.so") - ?? TryLoad("cu11.7.1/libllama.so") + /// + /// Get the xla flag for native library name. + /// + /// + private static string GetAvxFlag() + { + AvxLevel level = AvxLevel.None; +#if NET6_0_OR_GREATER + if (Avx.IsSupported) level = AvxLevel.Avx; + if (Avx2.IsSupported) level = AvxLevel.Avx2; #if NET8_0_OR_GREATER - ?? TryLoad("avx512/libllama.so", System.Runtime.Intrinsics.X86.Avx512.IsSupported) + if(Avx512F.IsSupported) level = AvxLevel.Avx512; #endif - ?? TryLoad("avx2/libllama.so", System.Runtime.Intrinsics.X86.Avx2.IsSupported) - ?? TryLoad("avx/libllama.so", System.Runtime.Intrinsics.X86.Avx.IsSupported) - ?? IntPtr.Zero; - } - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + return level switch { - return TryLoad("runtimes/macos-arm64/libllama.dylib", System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported) - ?? TryLoad("runtimes/macos-x86_64/libllama.dylib") - ?? IntPtr.Zero; - } + AvxLevel.None => "", + AvxLevel.Avx => "-avx", + AvxLevel.Avx2 => "-avx2", + AvxLevel.Avx512 => "-avx512", + }; +#else + return string.Empty; #endif - - return IntPtr.Zero; + } #if NET6_0_OR_GREATER - // Try to load a DLL from the path if supported. Returns null if nothing is loaded. - static IntPtr? TryLoad(string path, bool supported = true) + private static IntPtr LLamaImportResolver(string name, Assembly assembly, DllImportSearchPath? searchPath) + { + IntPtr handle = IntPtr.Zero; + if(!name.Equals(libraryName)) { - if (!supported) - return null; - - if (NativeLibrary.TryLoad(path, out var handle)) - return handle; + return NativeLibrary.Load(name, assembly, searchPath); + } - return null; + string libraryPath = string.Empty; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var avxFlag = GetAvxFlag(); + // check cuda + var cudaVersion = GetCudaVersion(); + if(cudaVersion == 11) + { + libraryPath = $"runtimes/win-x64/native/libllama-cuda11{avxFlag}.dll"; + } + else if (cudaVersion == 12) + { + libraryPath = $"runtimes/win-x64/native/libllama-cuda12{avxFlag}.dll"; + } + else if(cudaVersion == -1) // cpu version + { + libraryPath = $"runtimes/win-x64/native/libllama{avxFlag}.dll"; + } + else + { + throw new NotImplementedException($"Cuda version {cudaVersion} has not been supported, please compile dll yourself or open an issue in LLamaSharp."); + } } -#endif + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + var avxFlag = GetAvxFlag(); + // check cuda + var cudaVersion = GetCudaVersion(); + if (cudaVersion == 11) + { + libraryPath = $"runtimes/linux-x64/native/libllama-cuda11{avxFlag}.so"; + } + else if (cudaVersion == 12) + { + libraryPath = $"runtimes/linux-x64/native/libllama-cuda12{avxFlag}.so"; + } + else if (cudaVersion == -1) // cpu version + { + libraryPath = $"runtimes/linux-x64/native/libllama{avxFlag}.so"; + } + else + { + throw new NotImplementedException($"Cuda version {cudaVersion} has not been supported, please compile dll yourself or open an issue in LLamaSharp."); + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + if (System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported) + { + libraryPath = $"runtimes/osx-arm64/native/libllama.dylib"; + } + else + { + libraryPath = $"runtimes/osx-x64/native/libllama.dylib"; + } + } + + NativeLibrary.TryLoad(libraryPath, assembly, searchPath, out handle); + return handle; } +#endif + private const string libraryName = "libllama"; @@ -574,5 +637,13 @@ namespace LLama.Native /// [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] public static extern int llama_set_n_threads(SafeLLamaContextHandle ctx, uint n_threads, uint n_threads_batch); + + private enum AvxLevel + { + None = 0, + Avx = 1, + Avx2 = 2, + Avx512 = 3 + } } } diff --git a/README.md b/README.md index ea72e8a4..397f641e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **The C#/.NET binding of [llama.cpp](https://github.com/ggerganov/llama.cpp). It provides higher-level APIs to inference the LLaMA Models and deploy it on local device with C#/.NET. It works on -both Windows, Linux and MAC without requirment for compiling llama.cpp yourself. Even without GPU or not enought GPU memory, you can still apply LLaMA models well with this repo. 🤗** +both Windows, Linux and MAC without requirment for compiling llama.cpp yourself. Even without GPU or not enough GPU memory, you can still apply LLaMA models well with this repo. 🤗** **Furthermore, it provides integrations with other projects such as [semantic-kernel](https://github.com/microsoft/semantic-kernel), [kernel-memory](https://github.com/microsoft/kernel-memory) and [BotSharp](https://github.com/SciSharp/BotSharp) to provide higher-level applications.**