Merge d4cc122317 into b25f93b86d

LLama.Examples/Program.cs

@@ -1,5 +1,6 @@
 using LLama.Native;
 using Spectre.Console;
+using System.Runtime.InteropServices;
 
 AnsiConsole.MarkupLineInterpolated(
     $"""
@@ -16,23 +17,24 @@ AnsiConsole.MarkupLineInterpolated(
 
     """);
 
-// Configure native library to use. This must be done before any other llama.cpp methods are called!
-NativeLibraryConfig
-   .Instance
-   .WithCuda();
-
 // Configure logging. Change this to `true` to see log messages from llama.cpp
 var showLLamaCppLogs = false;
 NativeLibraryConfig
-   .Instance
+   .All
    .WithLogCallback((level, message) =>
-    {
-        if (showLLamaCppLogs)
-            Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
-    });
+   {
+       if (showLLamaCppLogs)
+           Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
+   });
+
+// Configure native library to use. This must be done before any other llama.cpp methods are called!
+NativeLibraryConfig
+   .All
+   .WithCuda()
+   //.WithAutoDownload() // An experimental feature
+   .DryRun(out var loadedllamaLibrary, out var loadedLLavaLibrary);
 
 // Calling this method forces loading to occur now.
 NativeApi.llama_empty_call();
 
-await ExampleRunner.Run();
-
+await ExampleRunner.Run();

LLama/Abstractions/INativeLibrary.cs

@@ -0,0 +1,29 @@
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions
+{
+    /// <summary>
+    /// Descriptor of a native library.
+    /// </summary>
+    public interface INativeLibrary
+    {
+        /// <summary>
+        /// Metadata of this library.
+        /// </summary>
+        NativeLibraryMetadata? Metadata { get; }
+
+        /// <summary>
+        /// Prepare the native library file and returns the local path of it.
+        /// If it's a relative path, LLamaSharp will search the path in the search directies you set.
+        /// </summary>
+        /// <param name="systemInfo">The system information of the current machine.</param>
+        /// <param name="logCallback">The log callback.</param>
+        /// <returns>
+        /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array.
+        /// </returns>
+        IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null);
+    }
+}

LLama/Abstractions/INativeLibrarySelectingPolicy.cs

@@ -0,0 +1,24 @@
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// Decides the selected native library that should be loaded according to the configurations.
+    /// </summary>
+    public interface INativeLibrarySelectingPolicy
+    {
+        /// <summary>
+        /// Select the native library.
+        /// </summary>
+        /// <param name="description"></param>
+        /// <param name="systemInfo">The system information of the current machine.</param>
+        /// <param name="logCallback">The log callback.</param>
+        /// <returns>The information of the selected native library files, in order by priority from the beginning to the end.</returns>
+        IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null);
+    }
+#endif
+}

LLama/LLamaSharp.csproj

@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net6.0;net7.0;net8.0</TargetFrameworks>
+    <TargetFrameworks>net6.0;net7.0;net8.0;netstandard2.0</TargetFrameworks>
     <RootNamespace>LLama</RootNamespace>
     <Nullable>enable</Nullable>
-    <LangVersion>10</LangVersion>
+    <LangVersion>12</LangVersion>
     <Platforms>AnyCPU;x64;Arm64</Platforms>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
 

LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs

@@ -0,0 +1,69 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <inheritdoc/>
+    public class DefaultNativeLibrarySelectingPolicy: INativeLibrarySelectingPolicy
+    {
+        /// <inheritdoc/>
+        public IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            List<INativeLibrary> results = new();
+
+            // Show the configuration we're working with
+            Log(description.ToString(), LLamaLogLevel.Info, logCallback);
+
+            // If a specific path is requested, only use it, no fall back.
+            if (!string.IsNullOrEmpty(description.Path))
+            {
+                yield return new NativeLibraryFromPath(description.Path);
+            }
+            else
+            {
+                if (description.UseCuda)
+                {
+                    yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck);
+                }
+
+                if(!description.UseCuda || description.AllowFallback)
+                {
+                    if (description.AllowFallback)
+                    {
+                        // Try all of the AVX levels we can support.
+                        if (description.AvxLevel >= AvxLevel.Avx512)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck);
+
+                        if (description.AvxLevel >= AvxLevel.Avx2)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck);
+
+                        if (description.AvxLevel >= AvxLevel.Avx)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck);
+
+                        yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck);
+                    }
+                    else
+                    {
+                        yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck);
+                    }
+                }
+
+                if(systemInfo.OSPlatform == OSPlatform.OSX || description.AllowFallback)
+                {
+                    yield return new NativeLibraryWithMacOrFallback(description.Library, description.SkipCheck);
+                }
+            }
+        }
+
+        private void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            if (!message.EndsWith("\n"))
+                message += "\n";
+
+            logCallback?.Invoke(level, message);
+        }
+    }
+#endif
+}

LLama/Native/Load/NativeLibraryConfig.cs

@@ -0,0 +1,618 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using LLama.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// Allows configuration of the native llama.cpp libraries to load and use.
+    /// All configuration must be done before using **any** other LLamaSharp methods!
+    /// </summary>
+    public sealed partial class NativeLibraryConfig
+    {
+        private string? _libraryPath;
+
+        private bool _useCuda = true;
+        private AvxLevel _avxLevel;
+        private bool _allowFallback = true;
+        private bool _skipCheck = false;
+
+        /// <summary>
+        /// search directory -> priority level, 0 is the lowest.
+        /// </summary>
+        private readonly List<string> _searchDirectories = new List<string>();
+
+        internal INativeLibrarySelectingPolicy SelectingPolicy { get; private set; } = new DefaultNativeLibrarySelectingPolicy();
+
+        #region configurators
+        /// <summary>
+        /// Load a specified native library as backend for LLamaSharp.
+        /// When this method is called, all the other configurations will be ignored.
+        /// </summary>
+        /// <param name="libraryPath">The full path to the native library to load.</param>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig WithLibrary(string? libraryPath)
+        {
+            ThrowIfLoaded();
+
+            _libraryPath = libraryPath;
+            return this;
+        }
+
+        /// <summary>
+        /// Configure whether to use cuda backend if possible. Default is true.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig WithCuda(bool enable = true)
+        {
+            ThrowIfLoaded();
+
+            _useCuda = enable;
+            return this;
+        }
+
+        /// <summary>
+        /// Configure the prefferred avx support level of the backend. 
+        /// Default value is detected automatically due to your operating system.
+        /// </summary>
+        /// <param name="level"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig WithAvx(AvxLevel level)
+        {
+            ThrowIfLoaded();
+
+            _avxLevel = level;
+            return this;
+        }
+
+        /// <summary>
+        /// Configure whether to allow fallback when there's no match for preferred settings. Default is true.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig WithAutoFallback(bool enable = true)
+        {
+            ThrowIfLoaded();
+
+            _allowFallback = enable;
+            return this;
+        }
+
+        /// <summary>
+        /// Whether to skip the check when you don't allow fallback. This option 
+        ///  may be useful under some complex conditions. For example, you're sure 
+        ///  you have your cublas configured but LLamaSharp take it as invalid by mistake. Default is false;
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfig SkipCheck(bool enable = true)
+        {
+            ThrowIfLoaded();
+
+            _skipCheck = enable;
+            return this;
+        }
+
+        /// <summary>
+        /// Add self-defined search directories. Note that the file structure of the added 
+        /// directories must be the same as the default directory. Besides, the directory 
+        /// won't be used recursively.
+        /// </summary>
+        /// <param name="directories"></param>
+        /// <returns></returns>
+        public NativeLibraryConfig WithSearchDirectories(IEnumerable<string> directories)
+        {
+            ThrowIfLoaded();
+
+            _searchDirectories.AddRange(directories);
+            return this;
+        }
+
+        /// <summary>
+        /// Add self-defined search directories. Note that the file structure of the added 
+        /// directories must be the same as the default directory. Besides, the directory 
+        /// won't be used recursively.
+        /// </summary>
+        /// <param name="directory"></param>
+        /// <returns></returns>
+        public NativeLibraryConfig WithSearchDirectory(string directory)
+        {
+            ThrowIfLoaded();
+
+            _searchDirectories.Add(directory);
+            return this;
+        }
+
+        /// <summary>
+        /// Set the policy which decides how to select the desired native libraries and order them by priority. 
+        /// By default we use <see cref="DefaultNativeLibrarySelectingPolicy"/>.
+        /// </summary>
+        /// <param name="policy"></param>
+        /// <returns></returns>
+        public NativeLibraryConfig WithSelectingPolicy(INativeLibrarySelectingPolicy policy)
+        {
+            ThrowIfLoaded();
+
+            SelectingPolicy = policy;
+            return this;
+        }
+
+        #endregion
+
+        internal Description CheckAndGatherDescription()
+        {
+            if (_allowFallback && _skipCheck)
+                throw new ArgumentException("Cannot skip the check when fallback is allowed.");
+
+            var path = _libraryPath;
+
+
+            return new Description(
+                path,
+                NativeLibraryName,
+                _useCuda,
+                _avxLevel,
+                _allowFallback,
+                _skipCheck,
+                _searchDirectories.Concat(new[] { "./" }).ToArray()
+            );
+        }
+
+        internal static string AvxLevelToString(AvxLevel level)
+        {
+            return level switch
+            {
+                AvxLevel.None => string.Empty,
+                AvxLevel.Avx => "avx",
+                AvxLevel.Avx2 => "avx2",
+                AvxLevel.Avx512 => "avx512",
+                _ => throw new ArgumentException($"Unknown AvxLevel '{level}'")
+            };
+        }
+
+        /// <summary>
+        /// Private constructor prevents new instances of this class being created
+        /// </summary>
+        private NativeLibraryConfig(NativeLibraryName nativeLibraryName)
+        {
+            NativeLibraryName = nativeLibraryName;
+
+            // Automatically detect the highest supported AVX level
+            if (System.Runtime.Intrinsics.X86.Avx.IsSupported)
+                _avxLevel = AvxLevel.Avx;
+            if (System.Runtime.Intrinsics.X86.Avx2.IsSupported)
+                _avxLevel = AvxLevel.Avx2;
+
+            if (CheckAVX512())
+                _avxLevel = AvxLevel.Avx512;
+        }
+
+        private static bool CheckAVX512()
+        {
+            if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported)
+                return false;
+
+            // ReSharper disable UnusedVariable (ebx is used when < NET8)
+            var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0);
+            // ReSharper restore UnusedVariable
+
+            var vnni = (ecx & 0b_1000_0000_0000) != 0;
+
+#if NET8_0_OR_GREATER
+            var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported;
+            var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported;
+            var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported;
+#else
+            var f = (ebx & (1 << 16)) != 0;
+            var bw = (ebx & (1 << 30)) != 0;
+            var vbmi = (ecx & 0b_0000_0000_0010) != 0;
+#endif
+
+            return vnni && vbmi && bw && f;
+        }
+
+        /// <summary>
+        /// The description of the native library configurations that's already specified.
+        /// </summary>
+        /// <param name="Path"></param>
+        /// <param name="Library"></param>
+        /// <param name="UseCuda"></param>
+        /// <param name="AvxLevel"></param>
+        /// <param name="AllowFallback"></param>
+        /// <param name="SkipCheck"></param>
+        /// <param name="SearchDirectories"></param>
+        public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, 
+            string[] SearchDirectories)
+        {
+            /// <inheritdoc/>
+            public override string ToString()
+            {
+                string avxLevelString = AvxLevel switch
+                {
+                    AvxLevel.None => "NoAVX",
+                    AvxLevel.Avx => "AVX",
+                    AvxLevel.Avx2 => "AVX2",
+                    AvxLevel.Avx512 => "AVX512",
+                    _ => "Unknown"
+                };
+
+                string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }";
+
+                return $"NativeLibraryConfig Description:\n" +
+                       $"- LibraryName: {Library}\n" +
+                       $"- Path: '{Path}'\n" +
+                       $"- PreferCuda: {UseCuda}\n" +
+                       $"- PreferredAvxLevel: {avxLevelString}\n" +
+                       $"- AllowFallback: {AllowFallback}\n" +
+                       $"- SkipCheck: {SkipCheck}\n" +
+                       $"- SearchDirectories and Priorities: {searchDirectoriesString}";
+            }
+        }
+    }
+#endif
+
+    public sealed partial class NativeLibraryConfig
+    {
+        /// <summary>
+        /// Set configurations for all the native libraries, including LLama and LLava
+        /// </summary>
+        [Obsolete("Please use NativeLibraryConfig.All instead, or set configurations for NativeLibraryConfig.LLama and NativeLibraryConfig.LLavaShared respectively.")]
+        public static NativeLibraryConfigContainer Instance => All;
+
+        /// <summary>
+        /// Set configurations for all the native libraries, including LLama and LLava
+        /// </summary>
+        public static NativeLibraryConfigContainer All { get; }
+
+        /// <summary>
+        /// Configuration for LLama native library
+        /// </summary>
+        public static NativeLibraryConfig LLama { get; }
+
+        /// <summary>
+        /// Configuration for LLava native library
+        /// </summary>
+        public static NativeLibraryConfig LLava { get; }
+
+
+        /// <summary>
+        /// The current version.
+        /// </summary>
+        public static string CurrentVersion => VERSION; // This should be changed before publishing new version. TODO: any better approach?
+
+        private const string COMMIT_HASH = "f7001c";
+        private const string VERSION = "master";
+
+        /// <summary>
+        /// Get the llama.cpp commit hash of the current version.
+        /// </summary>
+        /// <returns></returns>
+        public static string GetNativeLibraryCommitHash() => COMMIT_HASH;
+
+        static NativeLibraryConfig()
+        {
+            LLama = new(NativeLibraryName.LLama);
+            LLava = new(NativeLibraryName.LLava);
+            All = new(LLama, LLava);
+        }
+
+#if NETSTANDARD2_0
+        private NativeLibraryConfig(NativeLibraryName nativeLibraryName)
+        {
+            NativeLibraryName = nativeLibraryName;
+        }
+#endif
+
+        /// <summary>
+        /// Check if the native library has already been loaded. Configuration cannot be modified if this is true.
+        /// </summary>
+        public bool LibraryHasLoaded { get; internal set; }
+
+        internal NativeLibraryName NativeLibraryName { get; }
+
+        internal NativeLogConfig.LLamaLogCallback? LogCallback { get; private set; } = null;
+
+        private void ThrowIfLoaded()
+        {
+            if (LibraryHasLoaded)
+                throw new InvalidOperationException("The library has already loaded, you can't change the configurations. " +
+                    "Please finish the configuration setting before any call to LLamaSharp native APIs." +
+                    "Please use NativeLibraryConfig.DryRun if you want to see whether it's loaded " +
+                    "successfully but still have chance to modify the configurations.");
+        }
+
+        /// <summary>
+        /// Set the log callback that will be used for all llama.cpp log messages
+        /// </summary>
+        /// <param name="callback"></param>
+        /// <exception cref="NotImplementedException"></exception>
+        public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback)
+        {
+            ThrowIfLoaded();
+
+            LogCallback = callback;
+            return this;
+        }
+
+        /// <summary>
+        /// Set the log callback that will be used for all llama.cpp log messages
+        /// </summary>
+        /// <param name="logger"></param>
+        /// <exception cref="NotImplementedException"></exception>
+        public NativeLibraryConfig WithLogCallback(ILogger? logger)
+        {
+            ThrowIfLoaded();
+
+            // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead.
+            NativeLogConfig.llama_log_set(logger);
+
+            return this;
+        }
+
+        /// <summary>
+        /// Try to load the native library with the current configurations, 
+        /// but do not actually set it to <see cref="NativeApi"/>.
+        /// 
+        /// You can still modify the configuration after this calling but only before any call from <see cref="NativeApi"/>.
+        /// </summary>
+        /// <param name="loadedLibrary">
+        /// The loaded livrary. When the loading failed, this will be null. 
+        /// However if you are using .NET standard2.0, this will never return null.
+        /// </param>
+        /// <returns>Whether the running is successful.</returns>
+        public bool DryRun(out INativeLibrary? loadedLibrary)
+        {
+            LogCallback?.Invoke(LLamaLogLevel.Debug, $"Beginning dry run for {this.NativeLibraryName.GetLibraryName()}...");
+            return NativeLibraryUtils.TryLoadLibrary(this, out loadedLibrary) != IntPtr.Zero;
+        }
+    }
+
+    /// <summary>
+    /// A class to set same configurations to multiple libraries at the same time.
+    /// </summary>
+    public sealed class NativeLibraryConfigContainer
+    {
+        private NativeLibraryConfig[] _configs;
+
+        internal NativeLibraryConfigContainer(params NativeLibraryConfig[] configs)
+        {
+            _configs = configs;
+        }
+
+        #region configurators
+
+#if NET6_0_OR_GREATER
+        /// <summary>
+        /// Load a specified native library as backend for LLamaSharp.
+        /// When this method is called, all the other configurations will be ignored.
+        /// </summary>
+        /// <param name="llamaPath">The full path to the llama library to load.</param>
+        /// <param name="llavaPath">The full path to the llava library to load.</param>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer WithLibrary(string? llamaPath, string? llavaPath)
+        {
+            foreach(var config in _configs)
+            {
+                if(config.NativeLibraryName == NativeLibraryName.LLama && llamaPath is not null)
+                {
+                    config.WithLibrary(llamaPath);
+                }
+                if(config.NativeLibraryName == NativeLibraryName.LLava && llavaPath is not null)
+                {
+                    config.WithLibrary(llavaPath);
+                }
+            }
+
+            return this;
+        }
+
+        /// <summary>
+        /// Configure whether to use cuda backend if possible.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer WithCuda(bool enable = true)
+        {
+            foreach(var config in _configs)
+            {
+                config.WithCuda(enable);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Configure the prefferred avx support level of the backend.
+        /// </summary>
+        /// <param name="level"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer WithAvx(AvxLevel level)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithAvx(level);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Configure whether to allow fallback when there's no match for preferred settings.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer WithAutoFallback(bool enable = true)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithAutoFallback(enable);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Whether to skip the check when you don't allow fallback. This option 
+        ///  may be useful under some complex conditions. For example, you're sure 
+        ///  you have your cublas configured but LLamaSharp take it as invalid by mistake.
+        /// </summary>
+        /// <param name="enable"></param>
+        /// <returns></returns>
+        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
+        public NativeLibraryConfigContainer SkipCheck(bool enable = true)
+        {
+            foreach (var config in _configs)
+            {
+                config.SkipCheck(enable);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Add self-defined search directories. Note that the file structure of the added 
+        /// directories must be the same as the default directory. Besides, the directory 
+        /// won't be used recursively.
+        /// </summary>
+        /// <param name="directories"></param>
+        /// <returns></returns>
+        public NativeLibraryConfigContainer WithSearchDirectories(IEnumerable<string> directories)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithSearchDirectories(directories);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Add self-defined search directories. Note that the file structure of the added 
+        /// directories must be the same as the default directory. Besides, the directory 
+        /// won't be used recursively.
+        /// </summary>
+        /// <param name="directory"></param>
+        /// <returns></returns>
+        public NativeLibraryConfigContainer WithSearchDirectory(string directory)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithSearchDirectory(directory);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Set the policy which decides how to select the desired native libraries and order them by priority. 
+        /// By default we use <see cref="DefaultNativeLibrarySelectingPolicy"/>.
+        /// </summary>
+        /// <param name="policy"></param>
+        /// <returns></returns>
+        public NativeLibraryConfigContainer WithSelectingPolicy(INativeLibrarySelectingPolicy policy)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithSelectingPolicy(policy);
+            }
+            return this;
+        }
+#endif
+
+        /// <summary>
+        /// Set the log callback that will be used for all llama.cpp log messages
+        /// </summary>
+        /// <param name="callback"></param>
+        /// <exception cref="NotImplementedException"></exception>
+        public NativeLibraryConfigContainer WithLogCallback(NativeLogConfig.LLamaLogCallback? callback)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithLogCallback(callback);
+            }
+            return this;
+        }
+
+        /// <summary>
+        /// Set the log callback that will be used for all llama.cpp log messages
+        /// </summary>
+        /// <param name="logger"></param>
+        /// <exception cref="NotImplementedException"></exception>
+        public NativeLibraryConfigContainer WithLogCallback(ILogger? logger)
+        {
+            foreach (var config in _configs)
+            {
+                config.WithLogCallback(logger);
+            }
+            return this;
+        }
+
+        #endregion
+
+        /// <summary>
+        /// Try to load the native library with the current configurations, 
+        /// but do not actually set it to <see cref="NativeApi"/>.
+        /// 
+        /// You can still modify the configuration after this calling but only before any call from <see cref="NativeApi"/>.
+        /// </summary>
+        /// <returns>Whether the running is successful.</returns>
+        public bool DryRun(out INativeLibrary? loadedLLamaNativeLibrary, out INativeLibrary? loadedLLavaNativeLibrary)
+        {
+            bool success = true;
+            foreach(var config in _configs)
+            {
+                success &= config.DryRun(out var loadedLibrary);
+                if(config.NativeLibraryName == NativeLibraryName.LLama)
+                {
+                    loadedLLamaNativeLibrary = loadedLibrary;
+                }
+                else if(config.NativeLibraryName == NativeLibraryName.LLava)
+                {
+                    loadedLLavaNativeLibrary = loadedLibrary;
+                }
+                else
+                {
+                    throw new Exception("Unknown native library config during the dry run.");
+                }
+            }
+            loadedLLamaNativeLibrary = loadedLLavaNativeLibrary = null;
+            return success;
+        }
+    }
+
+    /// <summary>
+    /// The name of the native library
+    /// </summary>
+    public enum NativeLibraryName
+    {
+        /// <summary>
+        /// The native library compiled from llama.cpp.
+        /// </summary>
+        LLama,
+        /// <summary>
+        /// The native library compiled from the LLaVA example of llama.cpp.
+        /// </summary>
+        LLava
+    }
+
+    internal static class LibraryNameExtensions
+    {
+        public static string GetLibraryName(this NativeLibraryName name)
+        {
+            switch (name)
+            {
+                case NativeLibraryName.LLama:
+                    return NativeApi.libraryName;
+                case NativeLibraryName.LLava:
+                    return NativeApi.llavaLibraryName;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(name), name, null);
+            }
+        }
+    }
+}

LLama/Native/Load/NativeLibraryFromPath.cs

@@ -0,0 +1,31 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+
+namespace LLama.Native
+{
+    /// <summary>
+    /// A native library specified with a local file path.
+    /// </summary>
+    public class NativeLibraryFromPath: INativeLibrary
+    {
+        private string _path;
+
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata => null;
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="path"></param>
+        public NativeLibraryFromPath(string path)
+        {
+            _path = path;
+        }
+        
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            return [_path];
+        }
+    }
+}

LLama/Native/Load/NativeLibraryMetadata.cs

@@ -0,0 +1,43 @@
+
+namespace LLama.Native
+{
+    /// <summary>
+    /// Information of a native library file.
+    /// </summary>
+    /// <param name="NativeLibraryName">Which kind of library it is.</param>
+    /// <param name="UseCuda">Whether it's compiled with cublas.</param>
+    /// <param name="AvxLevel">Which AvxLevel it's compiled with.</param>
+    public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel)
+    {
+        public override string ToString()
+        {
+            return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})";
+        }
+    }
+
+    /// <summary>
+    /// Avx support configuration
+    /// </summary>
+    public enum AvxLevel
+    {
+        /// <summary>
+        /// No AVX
+        /// </summary>
+        None,
+
+        /// <summary>
+        /// Advanced Vector Extensions (supported by most processors after 2011)
+        /// </summary>
+        Avx,
+
+        /// <summary>
+        /// AVX2 (supported by most processors after 2013)
+        /// </summary>
+        Avx2,
+
+        /// <summary>
+        /// AVX512 (supported by some processors after 2016, not widely supported)
+        /// </summary>
+        Avx512,
+    }
+}

LLama/Native/Load/NativeLibraryUtils.cs

@@ -0,0 +1,160 @@
+using LLama.Abstractions;
+using LLama.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+    internal static class NativeLibraryUtils
+    {
+        /// <summary>
+        /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible
+        /// </summary>
+        /// <returns>The library handle to unload later, or IntPtr.Zero if no library was loaded</returns>
+        internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLibrary? loadedLibrary)
+        {
+#if NET6_0_OR_GREATER
+            var description = config.CheckAndGatherDescription();
+            var systemInfo = SystemInfo.Get();
+            Log($"Loading library: '{config.NativeLibraryName.GetLibraryName()}'", LLamaLogLevel.Debug, config.LogCallback);
+
+            // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not)
+            NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var ext, out var libPrefix);
+            Log($"Detected OS Platform: '{systemInfo.OSPlatform}'", LLamaLogLevel.Info, config.LogCallback);
+            Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug, config.LogCallback);
+            Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug, config.LogCallback);
+            Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug, config.LogCallback);
+
+            // Set the flag to ensure this config can no longer be modified
+            config.LibraryHasLoaded = true;
+
+            // Show the configuration we're working with
+            Log(description.ToString(), LLamaLogLevel.Info, config.LogCallback);
+
+            // Get the libraries ordered by priority from the selecting policy.
+            var libraries = config.SelectingPolicy.Apply(description, systemInfo, config.LogCallback);
+
+            foreach (var library in libraries)
+            {
+                // Prepare the local library file and get the path.
+                var paths = library.Prepare(systemInfo, config.LogCallback);
+                foreach (var path in paths)
+                {
+                    Log($"Got relative library path '{path}' from local with {library.Metadata}, trying to load it...", LLamaLogLevel.Debug, config.LogCallback);
+
+                    var result = TryLoad(path, description.SearchDirectories, config.LogCallback);
+                    if (result != IntPtr.Zero)
+                    {
+                        loadedLibrary = library;
+                        return result;
+                    }
+                }
+            }
+
+            // If fallback is allowed, we will make the last try (the default system loading) when calling the native api.
+            // Otherwise we throw an exception here.
+            if (!description.AllowFallback)
+            {
+                throw new RuntimeError("Failed to load the native library. Please check the log for more information.");
+            }
+            loadedLibrary = null;
+#else
+            loadedLibrary = new UnknownNativeLibrary();
+#endif
+
+            Log($"No library was loaded before calling native apis. " +
+                $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning, config.LogCallback);
+            return IntPtr.Zero;
+
+#if NET6_0_OR_GREATER
+            // Try to load a DLL from the path.
+            // Returns null if nothing is loaded.
+            static IntPtr TryLoad(string path, IEnumerable<string> searchDirectories, NativeLogConfig.LLamaLogCallback? logCallback)
+            {
+                var fullPath = TryFindPath(path, searchDirectories);
+                Log($"Found full path file '{fullPath}' for relative path '{path}'", LLamaLogLevel.Debug, logCallback);
+                if (NativeLibrary.TryLoad(fullPath, out var handle))
+                {
+                    Log($"Successfully loaded '{fullPath}'", LLamaLogLevel.Info, logCallback);
+                    return handle;
+                }
+
+                Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info, logCallback);
+                return IntPtr.Zero;
+            }
+#endif
+        }
+
+        // Try to find the given file in any of the possible search paths
+        private static string TryFindPath(string filename, IEnumerable<string> searchDirectories)
+        {
+            // Try the configured search directories in the configuration
+            foreach (var path in searchDirectories)
+            {
+                var candidate = Path.Combine(path, filename);
+                if (File.Exists(candidate))
+                    return candidate;
+            }
+
+            // Try a few other possible paths
+            var possiblePathPrefix = new[] {
+                    AppDomain.CurrentDomain.BaseDirectory,
+                    Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? ""
+                };
+
+            foreach (var path in possiblePathPrefix)
+            {
+                var candidate = Path.Combine(path, filename);
+                if (File.Exists(candidate))
+                    return candidate;
+            }
+
+            return filename;
+        }
+
+        private static void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            if (!message.EndsWith("\n"))
+                message += "\n";
+
+            logCallback?.Invoke(level, message);
+        }
+
+#if NET6_0_OR_GREATER
+        public static void GetPlatformPathParts(OSPlatform platform, out string os, out string fileExtension, out string libPrefix)
+        {
+            if (platform == OSPlatform.Windows)
+            {
+                os = "win-x64";
+                fileExtension = ".dll";
+                libPrefix = "";
+                return;
+            }
+
+            if (platform == OSPlatform.Linux)
+            {
+                os = "linux-x64";
+                fileExtension = ".so";
+                libPrefix = "lib";
+                return;
+            }
+
+            if (platform == OSPlatform.OSX)
+            {
+                fileExtension = ".dylib";
+
+                os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported
+                    ? "osx-arm64"
+                    : "osx-x64";
+                libPrefix = "lib";
+            }
+            else
+            {
+                throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp.");
+            }
+        }
+#endif
+    }
+}

LLama/Native/Load/NativeLibraryWithAvx.cs

@@ -0,0 +1,62 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// A native library compiled with avx support but without cuda/cublas.
+    /// </summary>
+    public class NativeLibraryWithAvx : INativeLibrary
+    {
+        private NativeLibraryName _libraryName;
+        private AvxLevel _avxLevel;
+        private bool _skipCheck;
+
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata
+        {
+            get
+            {
+                return new NativeLibraryMetadata(_libraryName, false, _avxLevel);
+            }
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="libraryName"></param>
+        /// <param name="avxLevel"></param>
+        /// <param name="skipCheck"></param>
+        public NativeLibraryWithAvx(NativeLibraryName libraryName, AvxLevel avxLevel, bool skipCheck)
+        {
+            _libraryName = libraryName;
+            _avxLevel = avxLevel;
+            _skipCheck = skipCheck;
+        }
+
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            if (systemInfo.OSPlatform != OSPlatform.Windows && systemInfo.OSPlatform != OSPlatform.Linux && !_skipCheck)
+            {
+                // Not supported on systems other than Windows and Linux.
+                return [];
+            }
+            var path = GetAvxPath(systemInfo, _avxLevel, logCallback);
+            return path is null ? [] : [path];
+        }
+
+        private string? GetAvxPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
+            var avxStr = NativeLibraryConfig.AvxLevelToString(avxLevel);
+            if (!string.IsNullOrEmpty(avxStr))
+                avxStr += "/";
+            var relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+            return relativePath;
+        }
+    }
+#endif
+}

LLama/Native/Load/NativeLibraryWithCuda.cs

@@ -0,0 +1,79 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// A native library compiled with cublas/cuda.
+    /// </summary>
+    public class NativeLibraryWithCuda : INativeLibrary
+    {
+        private int _majorCudaVersion;
+        private NativeLibraryName _libraryName;
+        private AvxLevel _avxLevel;
+        private bool _skipCheck;
+
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata
+        {
+            get
+            {
+                return new NativeLibraryMetadata(_libraryName, true, _avxLevel);
+            }
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="majorCudaVersion"></param>
+        /// <param name="libraryName"></param>
+        /// <param name="skipCheck"></param>
+        public NativeLibraryWithCuda(int majorCudaVersion, NativeLibraryName libraryName, bool skipCheck)
+        {
+            _majorCudaVersion = majorCudaVersion;
+            _libraryName = libraryName;
+            _skipCheck = skipCheck;
+        }
+
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            // TODO: Avx level is ignored now, needs to be implemented in the future.
+            if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck)
+            {
+                if (_majorCudaVersion == -1 && _skipCheck)
+                {
+                    // Currently only 11 and 12 are supported.
+                    var cuda12LibraryPath = GetCudaPath(systemInfo, 12, logCallback);
+                    if (cuda12LibraryPath is not null)
+                    {
+                        yield return cuda12LibraryPath;
+                    }
+                    var cuda11LibraryPath = GetCudaPath(systemInfo, 11, logCallback);
+                    if (cuda11LibraryPath is not null)
+                    {
+                        yield return cuda11LibraryPath;
+                    }
+                }
+                else if (_majorCudaVersion != -1)
+                {
+                    var cudaLibraryPath = GetCudaPath(systemInfo, _majorCudaVersion, logCallback);
+                    if (cudaLibraryPath is not null)
+                    {
+                        yield return cudaLibraryPath;
+                    }
+                }
+            }
+        }
+
+        private string? GetCudaPath(SystemInfo systemInfo, int cudaVersion, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
+            var relativePath = $"runtimes/{os}/native/cuda{cudaVersion}/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+            return relativePath;
+        }
+    }
+#endif
+}

LLama/Native/Load/NativeLibraryWithMacOrFallback.cs

@@ -0,0 +1,64 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// A native library compiled on Mac, or fallbacks from all other libraries in the selection.
+    /// </summary>
+    public class NativeLibraryWithMacOrFallback : INativeLibrary
+    {
+        private NativeLibraryName _libraryName;
+        private bool _skipCheck;
+
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata
+        {
+            get
+            {
+                return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None);
+            }
+        }
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="libraryName"></param>
+        /// <param name="skipCheck"></param>
+        public NativeLibraryWithMacOrFallback(NativeLibraryName libraryName, bool skipCheck)
+        {
+            _libraryName = libraryName;
+            _skipCheck = skipCheck;
+        }
+
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            var path = GetPath(systemInfo, AvxLevel.None, logCallback);
+            return path is null ?[] : [path];
+        }
+
+        private string? GetPath(SystemInfo systemInfo, AvxLevel avxLevel, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
+            string relativePath;
+            if (systemInfo.OSPlatform == OSPlatform.OSX)
+            {
+                relativePath = $"runtimes/{os}/native/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+            }
+            else
+            {
+                var avxStr = NativeLibraryConfig.AvxLevelToString(AvxLevel.None);
+                if (!string.IsNullOrEmpty(avxStr))
+                    avxStr += "/";
+
+                relativePath = $"runtimes/{os}/native/{avxStr}{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+            }
+
+            return relativePath;
+        }
+    }
+#endif
+}

LLama/Native/Load/SystemInfo.cs

@@ -0,0 +1,129 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text.Json;
+
+namespace LLama.Native
+{
+    /// <summary>
+    /// Operating system information.
+    /// </summary>
+    /// <param name="OSPlatform"></param>
+    /// <param name="CudaMajorVersion"></param>
+    public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion)
+    {
+        /// <summary>
+        /// Get the system information of the current machine.
+        /// </summary>
+        /// <returns></returns>
+        /// <exception cref="PlatformNotSupportedException"></exception>
+        public static SystemInfo Get()
+        {
+            OSPlatform platform;
+            if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                platform = OSPlatform.Windows;
+            }
+            else if(RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            {
+                platform = OSPlatform.Linux;
+            }
+            else if(RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                platform = OSPlatform.OSX;
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
+
+            return new SystemInfo(platform, GetCudaMajorVersion());
+        }
+
+        #region CUDA version
+        private static int GetCudaMajorVersion()
+        {
+            string? cudaPath;
+            string version = "";
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH");
+                if (cudaPath is null)
+                {
+                    return -1;
+                }
+
+                //Ensuring cuda bin path is reachable. Especially for MAUI environment.
+                string cudaBinPath = Path.Combine(cudaPath, "bin");
+
+                if (Directory.Exists(cudaBinPath))
+                {
+                    AddDllDirectory(cudaBinPath);
+                }
+
+                version = GetCudaVersionFromPath(cudaPath);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+            {
+                // Try the default first
+                cudaPath = "/usr/local/bin/cuda";
+                version = GetCudaVersionFromPath(cudaPath);
+                if (string.IsNullOrEmpty(version))
+                {
+                    cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH");
+                    if (cudaPath is null)
+                    {
+                        return -1;
+                    }
+                    foreach (var path in cudaPath.Split(':'))
+                    {
+                        version = GetCudaVersionFromPath(Path.Combine(path, ".."));
+                        if (string.IsNullOrEmpty(version))
+                        {
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (string.IsNullOrEmpty(version))
+                return -1;
+
+            version = version.Split('.')[0];
+            if (int.TryParse(version, out var majorVersion))
+                return majorVersion;
+
+            return -1;
+        }
+
+        private static string GetCudaVersionFromPath(string cudaPath)
+        {
+            try
+            {
+                string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile));
+                using (JsonDocument document = JsonDocument.Parse(json))
+                {
+                    JsonElement root = document.RootElement;
+                    JsonElement cublasNode = root.GetProperty("libcublas");
+                    JsonElement versionNode = cublasNode.GetProperty("version");
+                    if (versionNode.ValueKind == JsonValueKind.Undefined)
+                    {
+                        return string.Empty;
+                    }
+                    return versionNode.GetString() ?? "";
+                }
+            }
+            catch (Exception)
+            {
+                return string.Empty;
+            }
+        }
+
+        // Put it here to avoid calling NativeApi when getting the cuda version.
+        [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
+        internal static extern int AddDllDirectory(string NewDirectory);
+
+        private const string cudaVersionFile = "version.json";
+        #endregion
+    }
+}

LLama/Native/Load/UnknownNativeLibrary.cs

@@ -0,0 +1,25 @@
+using LLama.Abstractions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace LLama.Native
+{
+    /// <summary>
+    /// When you are using .NET standard2.0, dynamic native library loading is not supported.
+    /// This class will be returned in <see cref="NativeLibraryConfig.DryRun(out INativeLibrary)"/>.
+    /// </summary>
+    public class UnknownNativeLibrary: INativeLibrary
+    {
+        /// <inheritdoc/>
+        public NativeLibraryMetadata? Metadata => null;
+
+        /// <inheritdoc/>
+        public IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null)
+        {
+            throw new NotSupportedException("This class is only a placeholder and should not be used to load native library.");
+        }
+    }
+}

LLama/Native/NativeApi.Load.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Runtime.InteropServices;
 using System.Text.Json;
 using System.Collections.Generic;
+using LLama.Abstractions;
 
 namespace LLama.Native
 {
@@ -18,7 +19,8 @@ namespace LLama.Native
             SetDllImportResolver();
 
             // Set flag to indicate that this point has been passed. No native library config can be done after this point.
-            NativeLibraryConfig.LibraryHasLoaded = true;
+            NativeLibraryConfig.LLama.LibraryHasLoaded = true;
+            NativeLibraryConfig.LLava.LibraryHasLoaded = true;
 
             // Immediately make a call which requires loading the llama DLL. This method call
             // can't fail unless the DLL hasn't been loaded.
@@ -38,8 +40,8 @@ namespace LLama.Native
             }
 
             // Now that the "loaded" flag is set configure logging in llama.cpp
-            if (NativeLibraryConfig.Instance.LogCallback != null)
-                NativeLogConfig.llama_log_set(NativeLibraryConfig.Instance.LogCallback);
+            if (NativeLibraryConfig.LLama.LogCallback != null)
+                NativeLogConfig.llama_log_set(NativeLibraryConfig.LLama.LogCallback);
 
             // Init llama.cpp backend
             llama_backend_init();
@@ -64,7 +66,7 @@ namespace LLama.Native
                         return _loadedLlamaHandle;
 
                     // Try to load a preferred library, based on CPU feature detection
-                    _loadedLlamaHandle = TryLoadLibraries(LibraryName.Llama);
+                    _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out _loadedLLamaLibrary);
                     return _loadedLlamaHandle;
                 }
 
@@ -75,7 +77,7 @@ namespace LLama.Native
                         return _loadedLlavaSharedHandle;
 
                     // Try to load a preferred library, based on CPU feature detection
-                    _loadedLlavaSharedHandle = TryLoadLibraries(LibraryName.LlavaShared);
+                    _loadedLlavaSharedHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLava, out _loadedLLavaLibrary);
                     return _loadedLlavaSharedHandle;
                 }
 
@@ -85,343 +87,26 @@ namespace LLama.Native
 #endif
         }
 
-        private static void Log(string message, LLamaLogLevel level)
-        {
-            if (!message.EndsWith("\n"))
-                message += "\n";
-
-            NativeLibraryConfig.Instance.LogCallback?.Invoke(level, message);
-        }
-
-        #region CUDA version
-        private static int GetCudaMajorVersion()
-        {
-            string? cudaPath;
-            string version = "";
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-            {
-                cudaPath = Environment.GetEnvironmentVariable("CUDA_PATH");
-                if (cudaPath is null)
-                {
-                    return -1;
-                }
-
-                //Ensuring cuda bin path is reachable. Especially for MAUI environment.
-                string cudaBinPath = Path.Combine(cudaPath, "bin");
-
-                if (Directory.Exists(cudaBinPath))
-                {
-                    AddDllDirectory(cudaBinPath);
-                }
-
-                version = GetCudaVersionFromPath(cudaPath);
-            }
-            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-            {
-                // Try the default first
-                cudaPath = "/usr/local/bin/cuda";
-                version = GetCudaVersionFromPath(cudaPath);
-                if (string.IsNullOrEmpty(version))
-                {
-                    cudaPath = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH");
-                    if (cudaPath is null)
-                    {
-                        return -1;
-                    }
-                    foreach (var path in cudaPath.Split(':'))
-                    {
-                        version = GetCudaVersionFromPath(Path.Combine(path, ".."));
-                        if (string.IsNullOrEmpty(version))
-                        {
-                            break;
-                        }
-                    }
-                }
-            }
-
-            if (string.IsNullOrEmpty(version))
-                return -1;
-
-            version = version.Split('.')[0];
-            if (int.TryParse(version, out var majorVersion))
-                return majorVersion;
-
-            return -1;
-        }
-
-        private static string GetCudaVersionFromPath(string cudaPath)
-        {
-            try
-            {
-                string json = File.ReadAllText(Path.Combine(cudaPath, cudaVersionFile));
-                using (JsonDocument document = JsonDocument.Parse(json))
-                {
-                    JsonElement root = document.RootElement;
-                    JsonElement cublasNode = root.GetProperty("libcublas");
-                    JsonElement versionNode = cublasNode.GetProperty("version");
-                    if (versionNode.ValueKind == JsonValueKind.Undefined)
-                    {
-                        return string.Empty;
-                    }
-                    return versionNode.GetString() ?? "";
-                }
-            }
-            catch (Exception)
-            {
-                return string.Empty;
-            }
-        }
-        #endregion
-
-#if NET6_0_OR_GREATER
-        private static IEnumerable<string> GetLibraryTryOrder(NativeLibraryConfig.Description configuration)
-        {
-            var loadingName = configuration.Library.GetLibraryName();
-            Log($"Loading library: '{loadingName}'", LLamaLogLevel.Debug);
-
-            // Get platform specific parts of the path (e.g. .so/.dll/.dylib, libName prefix or not)
-            GetPlatformPathParts(out var platform, out var os, out var ext, out var libPrefix);
-            Log($"Detected OS Platform: '{platform}'", LLamaLogLevel.Info);
-            Log($"Detected OS string: '{os}'", LLamaLogLevel.Debug);
-            Log($"Detected extension string: '{ext}'", LLamaLogLevel.Debug);
-            Log($"Detected prefix string: '{libPrefix}'", LLamaLogLevel.Debug);
-
-            if (configuration.UseCuda && (platform == OSPlatform.Windows || platform == OSPlatform.Linux))
-            {
-                var cudaVersion = GetCudaMajorVersion();
-                Log($"Detected cuda major version {cudaVersion}.", LLamaLogLevel.Info);
-
-                if (cudaVersion == -1 && !configuration.AllowFallback)
-                {
-                    // if check skipped, we just try to load cuda libraries one by one.
-                    if (configuration.SkipCheck)
-                    {
-                        yield return GetCudaLibraryPath(loadingName, "cuda12");
-                        yield return GetCudaLibraryPath(loadingName, "cuda11");
-                    }
-                    else
-                    {
-                        throw new RuntimeError("Configured to load a cuda library but no cuda detected on your device.");
-                    }
-                }
-                else if (cudaVersion == 11)
-                {
-                    yield return GetCudaLibraryPath(loadingName, "cuda11");
-                }
-                else if (cudaVersion == 12)
-                {
-                    yield return GetCudaLibraryPath(loadingName, "cuda12");
-                }
-                else if (cudaVersion > 0)
-                {
-                    throw new RuntimeError($"Cuda version {cudaVersion} hasn't been supported by LLamaSharp, please open an issue for it.");
-                }
-
-                // otherwise no cuda detected but allow fallback
-            }
-
-            // Add the CPU/Metal libraries
-            if (platform == OSPlatform.OSX)
-            {
-                // On Mac it's very simple, there's no AVX to consider.
-                yield return GetMacLibraryPath(loadingName);
-            }
-            else
-            {
-                if (configuration.AllowFallback)
-                {
-                    // Try all of the AVX levels we can support.
-                    if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx512)
-                        yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx512);
-
-                    if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx2)
-                        yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx2);
-
-                    if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx)
-                        yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.Avx);
-
-                    yield return GetAvxLibraryPath(loadingName, NativeLibraryConfig.AvxLevel.None);
-                }
-                else
-                {
-                    // Fallback is not allowed - use the exact specified AVX level
-                    yield return GetAvxLibraryPath(loadingName, configuration.AvxLevel);
-                }
-            }
-        }
-
-        private static string GetMacLibraryPath(string libraryName)
-        {
-            GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix);
-
-            return $"runtimes/{os}/native/{libPrefix}{libraryName}{fileExtension}";
-        }
-
-        /// <summary>
-        /// Given a CUDA version and some path parts, create a complete path to the library file
-        /// </summary>
-        /// <param name="libraryName">Library being loaded (e.g. "llama")</param>
-        /// <param name="cuda">CUDA version (e.g. "cuda11")</param>
-        /// <returns></returns>
-        private static string GetCudaLibraryPath(string libraryName, string cuda)
-        {
-            GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix);
-
-            return $"runtimes/{os}/native/{cuda}/{libPrefix}{libraryName}{fileExtension}";
-        }
-
         /// <summary>
-        /// Given an AVX level and some path parts, create a complete path to the library file
+        /// Get the loaded native library. If you are using netstandard2.0, it will always return null.
         /// </summary>
-        /// <param name="libraryName">Library being loaded (e.g. "llama")</param>
-        /// <param name="avx"></param>
+        /// <param name="name"></param>
         /// <returns></returns>
-        private static string GetAvxLibraryPath(string libraryName, NativeLibraryConfig.AvxLevel avx)
-        {
-            GetPlatformPathParts(out _, out var os, out var fileExtension, out var libPrefix);
-
-            var avxStr = NativeLibraryConfig.AvxLevelToString(avx);
-            if (!string.IsNullOrEmpty(avxStr))
-                avxStr += "/";
-
-            return $"runtimes/{os}/native/{avxStr}{libPrefix}{libraryName}{fileExtension}";
-        }
-
-        private static void GetPlatformPathParts(out OSPlatform platform, out string os, out string fileExtension, out string libPrefix)
-        {
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-            {
-                platform = OSPlatform.Windows;
-                os = "win-x64";
-                fileExtension = ".dll";
-                libPrefix = "";
-                return;
-            }
-            
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-            {
-                platform = OSPlatform.Linux;
-                os = "linux-x64";
-                fileExtension = ".so";
-                libPrefix = "lib";
-                return;
-            }
-            
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
-            {
-                platform = OSPlatform.OSX;
-                fileExtension = ".dylib";
-
-                os = System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported
-                    ? "osx-arm64"
-                    : "osx-x64";
-                libPrefix = "lib";
-            }
-            else
-            {
-                throw new RuntimeError("Your operating system is not supported, please open an issue in LLamaSharp.");
-            }
-        }
-#endif
-
-        /// <summary>
-        /// Try to load libllama/llava_shared, using CPU feature detection to try and load a more specialised DLL if possible
-        /// </summary>
-        /// <returns>The library handle to unload later, or IntPtr.Zero if no library was loaded</returns>
-        private static IntPtr TryLoadLibraries(LibraryName lib)
+        /// <exception cref="ArgumentException"></exception>
+        public static INativeLibrary? GetLoadedNativeLibrary(NativeLibraryName name)
         {
-#if NET6_0_OR_GREATER
-            var configuration = NativeLibraryConfig.CheckAndGatherDescription(lib);
-
-            // Set the flag to ensure the NativeLibraryConfig can no longer be modified
-            NativeLibraryConfig.LibraryHasLoaded = true;
-
-            // Show the configuration we're working with
-            Log(configuration.ToString(), LLamaLogLevel.Info);
-
-            // If a specific path is requested, load that or immediately fail
-            if (!string.IsNullOrEmpty(configuration.Path))
+            return name switch
             {
-                if (!NativeLibrary.TryLoad(configuration.Path, out var handle))
-                    throw new RuntimeError($"Failed to load the native library [{configuration.Path}] you specified.");
-
-                Log($"Successfully loaded the library [{configuration.Path}] specified by user", LLamaLogLevel.Info);
-                return handle;
-            }
-
-            // Get a list of locations to try loading (in order of preference)
-            var libraryTryLoadOrder = GetLibraryTryOrder(configuration);
-
-            foreach (var libraryPath in libraryTryLoadOrder)
-            {
-                var fullPath = TryFindPath(libraryPath);
-                Log($"Trying '{fullPath}'", LLamaLogLevel.Debug);
-
-                var result = TryLoad(fullPath);
-                if (result != IntPtr.Zero)
-                {
-                    Log($"Loaded '{fullPath}'", LLamaLogLevel.Info);
-                    return result;
-                }
-
-                Log($"Failed Loading '{fullPath}'", LLamaLogLevel.Info);
-            }
-
-            if (!configuration.AllowFallback)
-            {
-                throw new RuntimeError("Failed to load the library that match your rule, please" +
-                    " 1) check your rule." +
-                    " 2) try to allow fallback." +
-                    " 3) or open an issue if it's expected to be successful.");
-            }
-#endif
-
-            Log($"No library was loaded before calling native apis. " +
-                $"This is not an error under netstandard2.0 but needs attention with net6 or higher.", LLamaLogLevel.Warning);
-            return IntPtr.Zero;
-
-#if NET6_0_OR_GREATER
-            // Try to load a DLL from the path.
-            // Returns null if nothing is loaded.
-            static IntPtr TryLoad(string path)
-            {
-                if (NativeLibrary.TryLoad(path, out var handle))
-                    return handle;
-
-                return IntPtr.Zero;
-            }
-
-            // Try to find the given file in any of the possible search paths
-            string TryFindPath(string filename)
-            {
-                // Try the configured search directories in the configuration
-                foreach (var path in configuration.SearchDirectories)
-                {
-                    var candidate = Path.Combine(path, filename);
-                    if (File.Exists(candidate))
-                        return candidate;
-                }
-
-                // Try a few other possible paths
-                var possiblePathPrefix = new[] {
-                    AppDomain.CurrentDomain.BaseDirectory,
-                    Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? ""
-                };
-
-                foreach (var path in possiblePathPrefix)
-                {
-                    var candidate = Path.Combine(path, filename);
-                    if (File.Exists(candidate))
-                        return candidate;
-                }
-
-                return filename;
-            }
-#endif
+                NativeLibraryName.LLama => _loadedLLamaLibrary,
+                NativeLibraryName.LLava => _loadedLLavaLibrary,
+                _ => throw new ArgumentException($"Library name {name} is not found.")
+            };
         }
 
         internal const string libraryName = "llama";
-        internal const string llavaLibraryName = "llava_shared";        
-        private const string cudaVersionFile = "version.json";
+        internal const string llavaLibraryName = "llava_shared";
+
+        private static INativeLibrary? _loadedLLamaLibrary = null;
+        private static INativeLibrary? _loadedLLavaLibrary = null;
     }
 }

LLama/Native/NativeApi.cs

@@ -19,9 +19,6 @@ namespace LLama.Native
             llama_max_devices();
         }
 
-        [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)]
-        private static extern int AddDllDirectory(string NewDirectory);
-
         /// <summary>
         /// Get the maximum number of devices supported by llama.cpp
         /// </summary>

LLama/Native/NativeLibraryConfig.cs

@@ -1,332 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using Microsoft.Extensions.Logging;
-
-namespace LLama.Native
-{
-#if NET6_0_OR_GREATER
-    /// <summary>
-    /// Allows configuration of the native llama.cpp libraries to load and use.
-    /// All configuration must be done before using **any** other LLamaSharp methods!
-    /// </summary>
-    public sealed partial class NativeLibraryConfig
-    {
-        private string? _libraryPath;
-        private string? _libraryPathLLava;
-
-        private bool _useCuda = true;
-        private AvxLevel _avxLevel;
-        private bool _allowFallback = true;
-        private bool _skipCheck = false;
-
-        /// <summary>
-        /// search directory -> priority level, 0 is the lowest.
-        /// </summary>
-        private readonly List<string> _searchDirectories = new List<string>();
-
-        #region configurators
-        /// <summary>
-        /// Load a specified native library as backend for LLamaSharp.
-        /// When this method is called, all the other configurations will be ignored.
-        /// </summary>
-        /// <param name="llamaPath">The full path to the llama library to load.</param>
-        /// <param name="llavaPath">The full path to the llava library to load.</param>
-        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
-        public NativeLibraryConfig WithLibrary(string? llamaPath, string? llavaPath)
-        {
-            ThrowIfLoaded();
-
-            _libraryPath = llamaPath;
-            _libraryPathLLava = llavaPath;
-            return this;
-        }
-
-        /// <summary>
-        /// Configure whether to use cuda backend if possible.
-        /// </summary>
-        /// <param name="enable"></param>
-        /// <returns></returns>
-        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
-        public NativeLibraryConfig WithCuda(bool enable = true)
-        {
-            ThrowIfLoaded();
-
-            _useCuda = enable;
-            return this;
-        }
-
-        /// <summary>
-        /// Configure the prefferred avx support level of the backend.
-        /// </summary>
-        /// <param name="level"></param>
-        /// <returns></returns>
-        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
-        public NativeLibraryConfig WithAvx(AvxLevel level)
-        {
-            ThrowIfLoaded();
-
-            _avxLevel = level;
-            return this;
-        }
-
-        /// <summary>
-        /// Configure whether to allow fallback when there's no match for preferred settings.
-        /// </summary>
-        /// <param name="enable"></param>
-        /// <returns></returns>
-        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
-        public NativeLibraryConfig WithAutoFallback(bool enable = true)
-        {
-            ThrowIfLoaded();
-
-            _allowFallback = enable;
-            return this;
-        }
-
-        /// <summary>
-        /// Whether to skip the check when you don't allow fallback. This option 
-        ///  may be useful under some complex conditions. For example, you're sure 
-        ///  you have your cublas configured but LLamaSharp take it as invalid by mistake.
-        /// </summary>
-        /// <param name="enable"></param>
-        /// <returns></returns>
-        /// <exception cref="InvalidOperationException">Thrown if `LibraryHasLoaded` is true.</exception>
-        public NativeLibraryConfig SkipCheck(bool enable = true)
-        {
-            ThrowIfLoaded();
-
-            _skipCheck = enable;
-            return this;
-        }
-
-        /// <summary>
-        /// Add self-defined search directories. Note that the file structure of the added 
-        /// directories must be the same as the default directory. Besides, the directory 
-        /// won't be used recursively.
-        /// </summary>
-        /// <param name="directories"></param>
-        /// <returns></returns>
-        public NativeLibraryConfig WithSearchDirectories(IEnumerable<string> directories)
-        {
-            ThrowIfLoaded();
-
-            _searchDirectories.AddRange(directories);
-            return this;
-        }
-
-        /// <summary>
-        /// Add self-defined search directories. Note that the file structure of the added 
-        /// directories must be the same as the default directory. Besides, the directory 
-        /// won't be used recursively.
-        /// </summary>
-        /// <param name="directory"></param>
-        /// <returns></returns>
-        public NativeLibraryConfig WithSearchDirectory(string directory)
-        {
-            ThrowIfLoaded();
-
-            _searchDirectories.Add(directory);
-            return this;
-        }
-        #endregion
-
-        internal static Description CheckAndGatherDescription(LibraryName library)
-        {
-            if (Instance._allowFallback && Instance._skipCheck)
-                throw new ArgumentException("Cannot skip the check when fallback is allowed.");
-
-            var path = library switch
-            {
-                LibraryName.Llama => Instance._libraryPath,
-                LibraryName.LlavaShared => Instance._libraryPathLLava,
-                _ => throw new ArgumentException($"Unknown library name '{library}'", nameof(library)),
-            };
-
-            return new Description(
-                path,
-                library,
-                Instance._useCuda,
-                Instance._avxLevel,
-                Instance._allowFallback,
-                Instance._skipCheck,
-                Instance._searchDirectories.Concat(new[] { "./" }).ToArray()
-            );
-        }
-
-        internal static string AvxLevelToString(AvxLevel level)
-        {
-            return level switch
-            {
-                AvxLevel.None => string.Empty,
-                AvxLevel.Avx => "avx",
-                AvxLevel.Avx2 => "avx2",
-                AvxLevel.Avx512 => "avx512",
-                _ => throw new ArgumentException($"Unknown AvxLevel '{level}'")
-            };
-        }
-
-        /// <summary>
-        /// Private constructor prevents new instances of this class being created
-        /// </summary>
-        private NativeLibraryConfig()
-        {
-            // Automatically detect the highest supported AVX level
-            if (System.Runtime.Intrinsics.X86.Avx.IsSupported)
-                _avxLevel = AvxLevel.Avx;
-            if (System.Runtime.Intrinsics.X86.Avx2.IsSupported)
-                _avxLevel = AvxLevel.Avx2;
-
-            if (CheckAVX512())
-                _avxLevel = AvxLevel.Avx512;
-        }
-
-        private static bool CheckAVX512()
-        {
-            if (!System.Runtime.Intrinsics.X86.X86Base.IsSupported)
-                return false;
-
-            // ReSharper disable UnusedVariable (ebx is used when < NET8)
-            var (_, ebx, ecx, _) = System.Runtime.Intrinsics.X86.X86Base.CpuId(7, 0);
-            // ReSharper restore UnusedVariable
-
-            var vnni = (ecx & 0b_1000_0000_0000) != 0;
-
-#if NET8_0_OR_GREATER
-            var f = System.Runtime.Intrinsics.X86.Avx512F.IsSupported;
-            var bw = System.Runtime.Intrinsics.X86.Avx512BW.IsSupported;
-            var vbmi = System.Runtime.Intrinsics.X86.Avx512Vbmi.IsSupported;
-#else
-            var f = (ebx & (1 << 16)) != 0;
-            var bw = (ebx & (1 << 30)) != 0;
-            var vbmi = (ecx & 0b_0000_0000_0010) != 0;
-#endif
-
-            return vnni && vbmi && bw && f;
-        }
-
-        /// <summary>
-        /// Avx support configuration
-        /// </summary>
-        public enum AvxLevel
-        {
-            /// <summary>
-            /// No AVX
-            /// </summary>
-            None,
-
-            /// <summary>
-            /// Advanced Vector Extensions (supported by most processors after 2011)
-            /// </summary>
-            Avx,
-
-            /// <summary>
-            /// AVX2 (supported by most processors after 2013)
-            /// </summary>
-            Avx2,
-
-            /// <summary>
-            /// AVX512 (supported by some processors after 2016, not widely supported)
-            /// </summary>
-            Avx512,
-        }
-
-        internal record Description(string? Path, LibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, string[] SearchDirectories)
-        {
-            public override string ToString()
-            {
-                string avxLevelString = AvxLevel switch
-                {
-                    AvxLevel.None => "NoAVX",
-                    AvxLevel.Avx => "AVX",
-                    AvxLevel.Avx2 => "AVX2",
-                    AvxLevel.Avx512 => "AVX512",
-                    _ => "Unknown"
-                };
-
-                string searchDirectoriesString = "{ " + string.Join(", ", SearchDirectories) + " }";
-
-                return $"NativeLibraryConfig Description:\n" +
-                       $"- LibraryName: {Library}\n" +
-                       $"- Path: '{Path}'\n" +
-                       $"- PreferCuda: {UseCuda}\n" +
-                       $"- PreferredAvxLevel: {avxLevelString}\n" +
-                       $"- AllowFallback: {AllowFallback}\n" +
-                       $"- SkipCheck: {SkipCheck}\n" +
-                       $"- SearchDirectories and Priorities: {searchDirectoriesString}";
-            }
-        }
-    }
-#endif
-
-    public sealed partial class NativeLibraryConfig
-    {
-        /// <summary>
-        /// Get the config instance
-        /// </summary>
-        public static NativeLibraryConfig Instance { get; } = new();
-
-        /// <summary>
-        /// Check if the native library has already been loaded. Configuration cannot be modified if this is true.
-        /// </summary>
-        public static bool LibraryHasLoaded { get; internal set; }
-
-        internal NativeLogConfig.LLamaLogCallback? LogCallback;
-
-        private static void ThrowIfLoaded()
-        {
-            if (LibraryHasLoaded)
-                throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!");
-        }
-
-        /// <summary>
-        /// Set the log callback that will be used for all llama.cpp log messages
-        /// </summary>
-        /// <param name="callback"></param>
-        /// <exception cref="NotImplementedException"></exception>
-        public NativeLibraryConfig WithLogCallback(NativeLogConfig.LLamaLogCallback? callback)
-        {
-            ThrowIfLoaded();
-
-            LogCallback = callback;
-            return this;
-        }
-
-        /// <summary>
-        /// Set the log callback that will be used for all llama.cpp log messages
-        /// </summary>
-        /// <param name="logger"></param>
-        /// <exception cref="NotImplementedException"></exception>
-        public NativeLibraryConfig WithLogCallback(ILogger? logger)
-        {
-            ThrowIfLoaded();
-
-            // Redirect to llama_log_set. This will wrap the logger in a delegate and bind that as the log callback instead.
-            NativeLogConfig.llama_log_set(logger);
-
-            return this;
-        }
-    }
-
-    internal enum LibraryName
-    {
-        Llama,
-        LlavaShared
-    }
-
-    internal static class LibraryNameExtensions
-    {
-        public static string GetLibraryName(this LibraryName name)
-        {
-            switch (name)
-            {
-                case LibraryName.Llama:
-                    return NativeApi.libraryName;
-                case LibraryName.LlavaShared:
-                    return NativeApi.llavaLibraryName;
-                default:
-                    throw new ArgumentOutOfRangeException(nameof(name), name, null);
-            }
-        }
-    }
-}

LLama/Native/NativeLogConfig.cs

@@ -37,7 +37,7 @@ public static class NativeLogConfig
     public static void llama_log_set(LLamaLogCallback? logCallback)
 #pragma warning restore IDE1006 // Naming Styles
     {
-        if (NativeLibraryConfig.LibraryHasLoaded)
+        if (NativeLibraryConfig.LLama.LibraryHasLoaded)
         {
             // The library is loaded, just pass the callback directly to llama.cpp
             native_llama_log_set(logCallback);