using System; using System.Collections.Generic; using System.Linq; using LLama.Abstractions; using Microsoft.Extensions.Logging; namespace LLama.Native { #if NET6_0_OR_GREATER /// /// Allows configuration of the native llama.cpp libraries to load and use. /// All configuration must be done before using **any** other LLamaSharp methods! /// public sealed partial class NativeLibraryConfig { private string? _libraryPath; private bool _useCuda = true; private AvxLevel _avxLevel; private bool _allowFallback = true; private bool _skipCheck = false; /// /// search directory -> priority level, 0 is the lowest. /// private readonly List _searchDirectories = new List(); internal INativeLibrarySelectingPolicy SelectingPolicy { get; private set; } = new DefaultNativeLibrarySelectingPolicy(); #region configurators /// /// Load a specified native library as backend for LLamaSharp. /// When this method is called, all the other configurations will be ignored. /// /// The full path to the native library to load. /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithLibrary(string? libraryPath) { ThrowIfLoaded(); _libraryPath = libraryPath; return this; } /// /// Configure whether to use cuda backend if possible. Default is true. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithCuda(bool enable = true) { ThrowIfLoaded(); _useCuda = enable; return this; } /// /// Configure the prefferred avx support level of the backend. /// Default value is detected automatically due to your operating system. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithAvx(AvxLevel level) { ThrowIfLoaded(); _avxLevel = level; return this; } /// /// Configure whether to allow fallback when there's no match for preferred settings. Default is true. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithAutoFallback(bool enable = true) { ThrowIfLoaded(); _allowFallback = enable; return this; } /// /// Whether to skip the check when you don't allow fallback. This option /// may be useful under some complex conditions. For example, you're sure /// you have your cublas configured but LLamaSharp take it as invalid by mistake. Default is false; /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig SkipCheck(bool enable = true) { ThrowIfLoaded(); _skipCheck = enable; return this; } /// /// Add self-defined search directories. Note that the file stucture of the added /// directories must be the same as the default directory. Besides, the directory /// won't be used recursively. /// /// /// public NativeLibraryConfig WithSearchDirectories(IEnumerable directories) { ThrowIfLoaded(); _searchDirectories.AddRange(directories); return this; } /// /// Add self-defined search directories. Note that the file stucture of the added /// directories must be the same as the default directory. Besides, the directory /// won't be used recursively. /// /// /// public NativeLibraryConfig WithSearchDirectory(string directory) { ThrowIfLoaded(); _searchDirectories.Add(directory); return this; } /// /// Set the policy which decides how to select the desired native libraries and order them by priority. /// By default we use . /// /// /// public NativeLibraryConfig WithSelectingPolicy(INativeLibrarySelectingPolicy policy) { ThrowIfLoaded(); SelectingPolicy = policy; return this; } #endregion internal Description CheckAndGatherDescription() { if (_allowFallback && _skipCheck) throw new ArgumentException("Cannot skip the check when fallback is allowed."); var path = _libraryPath; return new Description( path, NativeLibraryName, _useCuda, _avxLevel, _allowFallback, _skipCheck, _searchDirectories.Concat(new[] { "./" }).ToArray() ); } internal static string AvxLevelToString(AvxLevel level) { return level switch { AvxLevel.None => string.Empty, AvxLevel.Avx => "avx", AvxLevel.Avx2 => "avx2", AvxLevel.Avx512 => "avx512", _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") }; } /// /// Private constructor prevents new instances of this class being created /// private NativeLibraryConfig(NativeLibraryName nativeLibraryName) { NativeLibraryName = nativeLibraryName; // Automatically detect the highest supported AVX level if (System.Runtime.Intrinsics.X86.Avx.IsSupported) _avxLevel = AvxLevel.Avx; if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) _avxLevel = AvxLevel.Avx2; if (CheckAVX512()) _avxLevel = AvxLevel.Avx512; } private static bool CheckAVX512() { if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported) return false; // ReSharper disable UnusedVariable (ebx is used when < NET8) var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0); // ReSharper restore UnusedVariable var vnni = (ecx & 0b_1000_0000_0000) != 0; #if NET8_0_OR_GREATER var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported; var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported; var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported; #else var f = (ebx & (1 << 16)) != 0; var bw = (ebx & (1 << 30)) != 0; var vbmi = (ecx & 0b_0000_0000_0010) != 0; #endif return vnni && vbmi && bw && f; } /// /// The description of the native library configurations that's already specified. /// /// /// /// /// /// /// /// public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories) { /// public override string ToString() { string avxLevelString = AvxLevel switch { AvxLevel.None => "NoAVX", AvxLevel.Avx => "AVX", AvxLevel.Avx2 => "AVX2", AvxLevel.Avx512 => "AVX512", _ => "Unknown" }; string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }"; return $"NativeLibraryConfig Description:\n" + $"- LibraryName: {Library}\n" + $"- Path: '{Path}'\n" + $"- PreferCuda: {UseCuda}\n" + $"- PreferredAvxLevel: {avxLevelString}\n" + $"- AllowFallback: {AllowFallback}\n" + $"- SkipCheck: {SkipCheck}\n" + $"- SearchDirectories and Priorities: {searchDirectoriesString}"; } } } #endif public sealed partial class NativeLibraryConfig { /// /// Set configurations for all the native libraries, including LLama and LLava /// [Obsolete("Please use NativeLibraryConfig.All instead, or set configurations for NativeLibraryConfig.LLama and NativeLibraryConfig.LLavaShared respectively.")] public static NativeLibraryConfigContainer Instance { get; } /// /// Set configurations for all the native libraries, including LLama and LLava /// public static NativeLibraryConfigContainer All { get; } /// /// Configuration for LLama native library /// public static NativeLibraryConfig LLama { get; } /// /// Configuration for LLava native library /// public static NativeLibraryConfig LLavaShared { get; } /// /// A dictionary mapping from version to corresponding llama.cpp commit hash. /// The version should be formatted int `[major].[minor].[patch]`. But there's an exceptance that you can /// use `master` as a version to get the llama.cpp commit hash from the master branch. /// public static Dictionary VersionMap { get; } = new Dictionary() // This value should be changed when we're going to publish new release. (any better approach?) { {"master", "f7001c"} }; /// /// The current version. /// public static readonly string CurrentVersion = "master"; // This should be changed before publishing new version. TODO: any better approach? static NativeLibraryConfig() { LLama = new(NativeLibraryName.Llama); LLavaShared = new(NativeLibraryName.LlavaShared); All = new(LLama, LLavaShared); Instance = All; } #if NETSTANDARD2_0 private NativeLibraryConfig(NativeLibraryName nativeLibraryName) { NativeLibraryName = nativeLibraryName; } #endif /// /// Check if the native library has already been loaded. Configuration cannot be modified if this is true. /// public bool LibraryHasLoaded { get; internal set; } internal NativeLibraryName NativeLibraryName { get; } internal NativeLogConfig.LLamaLogCallback? LogCallback { get; private set; } = null; private void ThrowIfLoaded() { if (LibraryHasLoaded) throw new InvalidOperationException("The library has already loaded, you can't change the configurations. " + "Please finish the configuration setting before any call to LLamaSharp native APIs." + "Please use NativeLibraryConfig.DryRun if you want to see whether it's loaded " + "successfully but still have chance to modify the configurations."); } /// /// Set the log callback that will be used for all llama.cpp log messages /// /// /// public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) { ThrowIfLoaded(); LogCallback = callback; return this; } /// /// Set the log callback that will be used for all llama.cpp log messages /// /// /// public NativeLibraryConfig WithLogCallback(ILogger? logger) { ThrowIfLoaded(); // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead. NativeLogConfig.llama_log_set(logger); return this; } /// /// Try to load the native library with the current configurations, /// but do not actually set it to . /// /// You can still modify the configuration after this calling but only before any call from . /// /// Whether the running is successful. public bool DryRun() { LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}..."); return NativeLibraryUtils.TryLoadLibrary(this) != IntPtr.Zero; } } /// /// A class to set same configurations to multiple libraries at the same time. /// public sealed class NativeLibraryConfigContainer { private NativeLibraryConfig[] _configs; /// /// All the configurations in this container. /// Please avoid calling this property explicitly, use /// and instead. /// public NativeLibraryConfig[] Configs => _configs; internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs) { _configs = configs; } #region configurators #if NET6_0_OR_GREATER /// /// Load a specified native library as backend for LLamaSharp. /// When this method is called, all the other configurations will be ignored. /// /// The full path to the llama library to load. /// The full path to the llava library to load. /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfigContainer WithLibrary(string? llamaPath, string? llavaPath) { foreach(var config in _configs) { if(config.NativeLibraryName == NativeLibraryName.Llama && llamaPath is not null) { config.WithLibrary(llamaPath); } if(config.NativeLibraryName == NativeLibraryName.LlavaShared && llavaPath is not null) { config.WithLibrary(llavaPath); } } return this; } /// /// Configure whether to use cuda backend if possible. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfigContainer WithCuda(bool enable = true) { foreach(var config in _configs) { config.WithCuda(enable); } return this; } /// /// Configure the prefferred avx support level of the backend. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfigContainer WithAvx(AvxLevel level) { foreach (var config in _configs) { config.WithAvx(level); } return this; } /// /// Configure whether to allow fallback when there's no match for preferred settings. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfigContainer WithAutoFallback(bool enable = true) { foreach (var config in _configs) { config.WithAutoFallback(enable); } return this; } /// /// Whether to skip the check when you don't allow fallback. This option /// may be useful under some complex conditions. For example, you're sure /// you have your cublas configured but LLamaSharp take it as invalid by mistake. /// /// /// /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfigContainer SkipCheck(bool enable = true) { foreach (var config in _configs) { config.SkipCheck(enable); } return this; } /// /// Add self-defined search directories. Note that the file stucture of the added /// directories must be the same as the default directory. Besides, the directory /// won't be used recursively. /// /// /// public NativeLibraryConfigContainer WithSearchDirectories(IEnumerable directories) { foreach (var config in _configs) { config.WithSearchDirectories(directories); } return this; } /// /// Add self-defined search directories. Note that the file stucture of the added /// directories must be the same as the default directory. Besides, the directory /// won't be used recursively. /// /// /// public NativeLibraryConfigContainer WithSearchDirectory(string directory) { foreach (var config in _configs) { config.WithSearchDirectory(directory); } return this; } /// /// Set the policy which decides how to select the desired native libraries and order them by priority. /// By default we use . /// /// /// public NativeLibraryConfigContainer WithSelectingPolicy(INativeLibrarySelectingPolicy policy) { foreach (var config in _configs) { config.WithSelectingPolicy(policy); } return this; } #endif /// /// Set the log callback that will be used for all llama.cpp log messages /// /// /// public NativeLibraryConfigContainer WithLogCallback(NativeLogConfig.LLamaLogCallback? callback) { foreach (var config in _configs) { config.WithLogCallback(callback); } return this; } /// /// Set the log callback that will be used for all llama.cpp log messages /// /// /// public NativeLibraryConfigContainer WithLogCallback(ILogger? logger) { foreach (var config in _configs) { config.WithLogCallback(logger); } return this; } #endregion /// /// Try to load the native library with the current configurations, /// but do not actually set it to . /// /// You can still modify the configuration after this calling but only before any call from . /// /// Whether the running is successful. public bool DryRun() { return _configs.All(config => config.DryRun()); } } /// /// The name of the native library /// public enum NativeLibraryName { /// /// The native library compiled from llama.cpp. /// Llama, /// /// The native library compiled from the LLaVA example of llama.cpp. /// LlavaShared } internal static class LibraryNameExtensions { public static string GetLibraryName(this NativeLibraryName name) { switch (name) { case NativeLibraryName.Llama: return NativeApi.libraryName; case NativeLibraryName.LlavaShared: return NativeApi.llavaLibraryName; default: throw new ArgumentOutOfRangeException(nameof(name), name, null); } } } }