using System; namespace LLama.Native { #if NET6_0_OR_GREATER /// /// A class about configurations when loading native libraries. /// Note that it could be configured only once before any call to llama model apis. /// public class NativeLibraryConfig { private static NativeLibraryConfig? instance; private static readonly object lockObject = new object(); public static NativeLibraryConfig Default { get { return GetInstance(); } } /// /// Whether there's already a config for native library. /// public static bool LibraryHasLoaded { get; internal set; } = false; private string _libraryPath; private bool _useCuda; private AvxLevel _avxLevel; private bool _allowFallback; private bool _skipCheck; private bool _logging; internal static NativeLibraryConfig GetInstance() { if (instance is null) { lock (lockObject) { if (instance is null) { instance = new NativeLibraryConfig(); } } } return instance; } /// /// Load a specified native library as backend for LLamaSharp. /// When this method is called, all the other configurations will be ignored. /// /// /// public NativeLibraryConfig WithLibrary(string libraryPath) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _libraryPath = libraryPath; return this; } /// /// Configure whether to use cuda backend if possible. /// /// /// /// public NativeLibraryConfig WithCuda(bool enable = true) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _useCuda = enable; return this; } /// /// Configure the prefferred avx support level of the backend. /// /// /// /// public NativeLibraryConfig WithAvx(AvxLevel level) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _avxLevel = level; return this; } /// /// Configure whether to allow fallback when there's not match for preffered settings. /// /// /// /// public NativeLibraryConfig WithAutoFallback(bool enable = true) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _allowFallback = enable; return this; } /// /// Whether to skip the check when you don't allow fallback. This option /// may be useful under some complex conditions. For example, you're sure /// you have your cublas configured but LLamaSharp take it as invalid by mistake. /// /// /// /// public NativeLibraryConfig SkipCheck(bool enable = true) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _skipCheck = enable; return this; } /// /// Whether to output the logs to console when loading the native library with your configuration. /// /// /// /// public NativeLibraryConfig WithLogs(bool enable = true) { if (LibraryHasLoaded) { throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); } _logging = enable; return this; } internal static Description CheckAndGatherDescription() { if (Default._allowFallback && Default._skipCheck) { throw new ArgumentException("Cannot skip the check when fallback is allowed."); } return new Description(Default._libraryPath, Default._useCuda, Default._avxLevel, Default._allowFallback, Default._skipCheck, Default._logging); } internal static string AvxLevelToString(AvxLevel level) { return level switch { AvxLevel.None => string.Empty, AvxLevel.Avx => "avx", AvxLevel.Avx2 => "avx2", #if NET8_0_OR_GREATER AvxLevel.Avx512 => "avx512" #endif _ => throw new ArgumentException($"Cannot recognize Avx level {level}") }; } private NativeLibraryConfig() { _libraryPath = string.Empty; _useCuda = true; _avxLevel = AvxLevel.Avx2; _allowFallback = true; _skipCheck = false; _logging = false; } /// /// Avx support configuration /// public enum AvxLevel { /// None = 0, /// Avx = 1, /// Avx2 = 2, #if NET8_0_OR_GREATER /// Avx512 = 3, #endif } internal record Description(string Path = "", bool UseCuda = true, AvxLevel AvxLevel = AvxLevel.Avx2, bool AllowFallback = true, bool SkipCheck = false, bool Logging = false); } #endif }