scisharp
/
LLamaSharp
Not watched
1
0
Fork 0
Code
Releases 17 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Merge branch 'pr/268' into RuntimeDetection

tags/v0.8.0
SignalRT 2 years ago
parent
9b2ca9cf8e b893c6f609
commit
091b8d58b3
18 changed files with 210 additions and 218 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      LLama.Examples/LLama.Examples.csproj
  2. +2
    -1
      LLama.Examples/NewVersion/GetEmbeddings.cs
  3. +3
    -1
      LLama.Examples/NewVersion/QuantizeModel.cs
  4. +40
    -95
      LLama.Examples/NewVersion/TestRunner.cs
  5. +2
    -2
      LLama.Web/Common/ModelOptions.cs
  6. +2
    -2
      LLama/Abstractions/IContextParams.cs
  7. +1
    -1
      LLama/Common/FixedSizeQueue.cs
  8. +32
    -69
      LLama/Common/ModelParams.cs
  9. +2
    -0
      LLama/Extensions/DictionaryExtensions.cs
  10. +2
    -0
      LLama/Extensions/EncodingExtensions.cs
  11. +1
    -1
      LLama/Extensions/IContextParamsExtensions.cs
  12. +2
    -0
      LLama/Extensions/IEnumerableExtensions.cs
  13. +2
    -0
      LLama/Extensions/KeyValuePairExtensions.cs
  14. +1
    -1
      LLama/Extensions/ListExtensions.cs
  15. +1
    -0
      LLama/LLamaSharp.csproj
  16. +1
    -1
      LLama/Native/LLamaContextParams.cs
  17. +114
    -43
      LLama/Native/NativeApi.cs
  18. +1
    -1
      README.md

+ 1
- 0
LLama.Examples/LLama.Examples.csproj View File

@@ -30,6 +30,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.0.0-beta4" />
<PackageReference Include="Spectre.Console" Version="0.47.0" />
</ItemGroup>

<ItemGroup>


+ 2
- 1
LLama.Examples/NewVersion/GetEmbeddings.cs View File

@@ -4,7 +4,7 @@ namespace LLama.Examples.NewVersion
{
public class GetEmbeddings
{
public static void Run()
public static Task Run()
{
Console.Write("Please input your model path: ");
var modelPath = Console.ReadLine();
@@ -23,6 +23,7 @@ namespace LLama.Examples.NewVersion
Console.WriteLine(string.Join(", ", embedder.GetEmbeddings(text)));
Console.WriteLine();
}
return Task.CompletedTask;
}
}
}

+ 3
- 1
LLama.Examples/NewVersion/QuantizeModel.cs View File

@@ -2,7 +2,7 @@
{
public class QuantizeModel
{
public static void Run()
public static Task Run()
{
Console.Write("Please input your original model path: ");
var inputPath = Console.ReadLine();
@@ -21,6 +21,8 @@
{
Console.WriteLine("Quantization failed!");
}

return Task.CompletedTask;
}
}
}

+ 40
- 95
LLama.Examples/NewVersion/TestRunner.cs View File

@@ -1,109 +1,54 @@
namespace LLama.Examples.NewVersion
using System.Linq.Expressions;
using Spectre.Console;

namespace LLama.Examples.NewVersion
{
public class NewVersionTestRunner
{
static Dictionary<string, Func<Task>> Examples = new Dictionary<string, Func<Task>>
{
{"Run a chat session without stripping the role names.", () => ChatSessionWithRoleName.Run()},
{"Run a chat session with the role names stripped.",()=> ChatSessionStripRoleName.Run()},
{"Interactive mode chat by using executor.",()=> InteractiveModeExecute.Run()},
{"Instruct mode chat by using executor.",()=> InstructModeExecute.Run()},
{"Stateless mode chat by using executor.",()=> StatelessModeExecute.Run()},
{"Load and save chat session.",()=> SaveAndLoadSession.Run()},
{"Load and save state of model and executor.",()=> LoadAndSaveState.Run()},
{"Get embeddings from LLama model.",()=> GetEmbeddings.Run()},
{"Quantize the model.",()=> QuantizeModel.Run()},
{"Automatic conversation.",()=> TalkToYourself.Run()},
{"Constrain response to json format using grammar.",()=> GrammarJsonResponse.Run()},
{"Semantic Kernel Prompt.",()=> SemanticKernelPrompt.Run()},
{"Semantic Kernel Chat.",()=> SemanticKernelChat.Run()},
{"Semantic Kernel Memory.",()=> SemanticKernelMemory.Run()},
{"Coding Assistant.",()=> CodingAssistant.Run()},
{"Batch Decoding.",()=> BatchedDecoding.Run()},
{"SK Kernel Memory.",()=> KernelMemory.Run()},
{"Exit", ()=> Task.CompletedTask}
};
public static async Task Run()
{
Console.WriteLine("================LLamaSharp Examples (New Version)==================\n");

Console.WriteLine("Please input a number to choose an example to run:");
Console.WriteLine("0: Run a chat session without stripping the role names.");
Console.WriteLine("1: Run a chat session with the role names stripped.");
Console.WriteLine("2: Interactive mode chat by using executor.");
Console.WriteLine("3: Instruct mode chat by using executor.");
Console.WriteLine("4: Stateless mode chat by using executor.");
Console.WriteLine("5: Load and save chat session.");
Console.WriteLine("6: Load and save state of model and executor.");
Console.WriteLine("7: Get embeddings from LLama model.");
Console.WriteLine("8: Quantize the model.");
Console.WriteLine("9: Automatic conversation.");
Console.WriteLine("10: Constrain response to json format using grammar.");
Console.WriteLine("11: Semantic Kernel Prompt.");
Console.WriteLine("12: Semantic Kernel Chat.");
Console.WriteLine("13: Semantic Kernel Memory.");
Console.WriteLine("14: Coding Assistant.");
Console.WriteLine("15: Batch Decoding.");
Console.WriteLine("16: SK Kernel Memory.");
AnsiConsole.Write(new Rule("LLamaSharp Examples"));

while (true)
{
Console.Write("\nYour choice: ");
int choice = int.Parse(Console.ReadLine());
var choice = AnsiConsole.Prompt(
new SelectionPrompt<string>()
.Title("Please choose[green] an example[/] to run: ")
.AddChoices(Examples.Keys));

if (choice == 0)
{
await ChatSessionWithRoleName.Run();
}
else if (choice == 1)
{
await ChatSessionStripRoleName.Run();
}
else if (choice == 2)
{
await InteractiveModeExecute.Run();
}
else if (choice == 3)
{
await InstructModeExecute.Run();
}
else if (choice == 4)
{
await StatelessModeExecute.Run();
}
else if (choice == 5)
{
await SaveAndLoadSession.Run();
}
else if (choice == 6)
{
await LoadAndSaveState.Run();
}
else if (choice == 7)
{
GetEmbeddings.Run();
}
else if (choice == 8)
{
QuantizeModel.Run();
}
else if (choice == 9)
{
await TalkToYourself.Run();
}
else if (choice == 10)
{
await GrammarJsonResponse.Run();
}
else if (choice == 11)
{
await SemanticKernelPrompt.Run();
}
else if (choice == 12)
{
await SemanticKernelChat.Run();
}
else if (choice == 13)
{
await SemanticKernelMemory.Run();
}
else if (choice == 14)
{
await CodingAssistant.Run();
}
else if (choice == 15)
{
await BatchedDecoding.Run();
}
else if (choice == 16)
{
await KernelMemory.Run();
}
else

if (Examples.TryGetValue(choice, out var example))
{
Console.WriteLine("Cannot parse your choice. Please select again.");
continue;
if (choice == "Exit")
{
break;
}
AnsiConsole.Write(new Rule(choice));
await example();
}
break;

AnsiConsole.Clear();
}
}
}


+ 2
- 2
LLama.Web/Common/ModelOptions.cs View File

@@ -17,9 +17,9 @@ namespace LLama.Web.Common
public int MaxInstances { get; set; }

/// <summary>
/// Model context size (n_ctx)
/// Model context size (n_ctx). Null to use value from model.
/// </summary>
public uint ContextSize { get; set; } = 512;
public uint? ContextSize { get; set; }

/// <summary>
/// the GPU that is used for scratch and small tensors


+ 2
- 2
LLama/Abstractions/IContextParams.cs View File

@@ -8,9 +8,9 @@ namespace LLama.Abstractions;
public interface IContextParams
{
/// <summary>
/// Model context size (n_ctx)
/// Model context size (n_ctx). Null to use value from model file.
/// </summary>
uint ContextSize { get; set; }
uint? ContextSize { get; set; }

/// <summary>
/// batch size for prompt processing (must be >=32 to use BLAS) (n_batch)


+ 1
- 1
LLama/Common/FixedSizeQueue.cs View File

@@ -43,7 +43,7 @@ namespace LLama.Common
/// <param name="data"></param>
public FixedSizeQueue(int size, IEnumerable<T> data)
{
#if !NETSTANDARD2_0
#if NET6_0_OR_GREATER
// Try to check the size without enumerating the entire IEnumerable. This may not be able to get the count,
// in which case we'll have to check later
if (data.TryGetNonEnumeratedCount(out var dataCount) && dataCount > size)


+ 32
- 69
LLama/Common/ModelParams.cs View File

@@ -12,105 +12,68 @@ namespace LLama.Common
public record ModelParams
: ILLamaParams
{
/// <summary>
/// Model context size (n_ctx)
/// </summary>
public uint ContextSize { get; set; } = 512;
/// <summary>
/// the GPU that is used for scratch and small tensors
/// </summary>
/// <inheritdoc />
public uint? ContextSize { get; set; }

/// <inheritdoc />
public int MainGpu { get; set; } = 0;

/// <summary>
/// Number of layers to run in VRAM / GPU memory (n_gpu_layers)
/// </summary>
/// <inheritdoc />
public int GpuLayerCount { get; set; } = 20;
/// <summary>
/// Seed for the random number generator (seed)
/// </summary>

/// <inheritdoc />
public uint Seed { get; set; } = 0xFFFFFFFF;
/// <summary>
/// Use f16 instead of f32 for memory kv (memory_f16)
/// </summary>

/// <inheritdoc />
public bool UseFp16Memory { get; set; } = true;
/// <summary>
/// Use mmap for faster loads (use_mmap)
/// </summary>

/// <inheritdoc />
public bool UseMemorymap { get; set; } = true;
/// <summary>
/// Use mlock to keep model in memory (use_mlock)
/// </summary>

/// <inheritdoc />
public bool UseMemoryLock { get; set; }
/// <summary>
/// Compute perplexity over the prompt (perplexity)
/// </summary>

/// <inheritdoc />
public bool Perplexity { get; set; }
/// <summary>
/// Model path (model)
/// </summary>

/// <inheritdoc />
public string ModelPath { get; set; }

/// <summary>
/// List of LoRAs to apply
/// </summary>
/// <inheritdoc />
public AdapterCollection LoraAdapters { get; set; } = new();

/// <summary>
/// base model path for the lora adapter (lora_base)
/// </summary>
/// <inheritdoc />
public string LoraBase { get; set; } = string.Empty;

/// <summary>
/// Number of threads (null = autodetect) (n_threads)
/// </summary>
/// <inheritdoc />
public uint? Threads { get; set; }

/// <summary>
/// Number of threads to use for batch processing (null = autodetect) (n_threads)
/// </summary>
/// <inheritdoc />
public uint? BatchThreads { get; set; }

/// <summary>
/// batch size for prompt processing (must be >=32 to use BLAS) (n_batch)
/// </summary>
/// <inheritdoc />
public uint BatchSize { get; set; } = 512;

/// <summary>
/// Whether to use embedding mode. (embedding) Note that if this is set to true,
/// The LLamaModel won't produce text response anymore.
/// </summary>
/// <inheritdoc />
public bool EmbeddingMode { get; set; }

/// <summary>
/// how split tensors should be distributed across GPUs.
/// </summary>
/// <remarks>"[ 3, 2 ]" will assign 60% of the data to GPU 0 and 40% to GPU 1.</remarks>
/// <inheritdoc />
[JsonConverter(typeof(TensorSplitsCollectionConverter))]
public TensorSplitsCollection TensorSplits { get; set; } = new();

/// <summary>
/// RoPE base frequency
/// </summary>
public float? RopeFrequencyBase { get; set; }
/// <inheritdoc />
public float? RopeFrequencyBase { get; set; }

/// <summary>
/// RoPE frequency scaling factor
/// </summary>
public float? RopeFrequencyScale { get; set; }
/// <inheritdoc />
public float? RopeFrequencyScale { get; set; }

/// <summary>
/// Use experimental mul_mat_q kernels
/// </summary>
public bool MulMatQ { get; set; }
/// <inheritdoc />
public bool MulMatQ { get; set; }

/// <summary>
/// Load vocab only (no weights)
/// </summary>
/// <inheritdoc />
public bool VocabOnly { get; set; }

/// <summary>
/// The encoding to use to convert text for the model
/// </summary>
/// <inheritdoc />
[JsonConverter(typeof(EncodingConverter))]
public Encoding Encoding { get; set; } = Encoding.UTF8;



+ 2
- 0
LLama/Extensions/DictionaryExtensions.cs View File

@@ -9,6 +9,8 @@ namespace LLama.Extensions
{
return GetValueOrDefaultImpl(dictionary, key, defaultValue);
}
#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER
#error Target framework not supported!
#endif

internal static TValue GetValueOrDefaultImpl<TKey, TValue>(IReadOnlyDictionary<TKey, TValue> dictionary, TKey key, TValue defaultValue)


+ 2
- 0
LLama/Extensions/EncodingExtensions.cs View File

@@ -15,6 +15,8 @@ internal static class EncodingExtensions
{
return GetCharCountImpl(encoding, bytes);
}
#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER
#error Target framework not supported!
#endif

internal static int GetCharsImpl(Encoding encoding, ReadOnlySpan<byte> bytes, Span<char> output)


+ 1
- 1
LLama/Extensions/IContextParamsExtensions.cs View File

@@ -21,7 +21,7 @@ namespace LLama.Extensions
public static void ToLlamaContextParams(this IContextParams @params, out LLamaContextParams result)
{
result = NativeApi.llama_context_default_params();
result.n_ctx = @params.ContextSize;
result.n_ctx = @params.ContextSize ?? 0;
result.n_batch = @params.BatchSize;
result.seed = @params.Seed;
result.f16_kv = @params.UseFp16Memory;


+ 2
- 0
LLama/Extensions/IEnumerableExtensions.cs View File

@@ -10,6 +10,8 @@ namespace LLama.Extensions
{
return TakeLastImpl(source, count);
}
#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER
#error Target framework not supported!
#endif

internal static IEnumerable<T> TakeLastImpl<T>(IEnumerable<T> source, int count)


+ 2
- 0
LLama/Extensions/KeyValuePairExtensions.cs View File

@@ -19,5 +19,7 @@ internal static class KeyValuePairExtensions
first = pair.Key;
second = pair.Value;
}
#elif !NET6_0_OR_GREATER && !NETSTANDARD2_1_OR_GREATER
#error Target framework not supported!
#endif
}

+ 1
- 1
LLama/Extensions/ListExtensions.cs View File

@@ -5,7 +5,7 @@ namespace LLama.Extensions
{
internal static class ListExtensions
{
#if NETSTANDARD2_0
#if !NET6_0_OR_GREATER
public static void EnsureCapacity<T>(this List<T> list, int capacity)
{
if (list.Capacity < capacity)


+ 1
- 0
LLama/LLamaSharp.csproj View File

@@ -47,6 +47,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="ManagedCuda" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
</ItemGroup>



+ 1
- 1
LLama/Native/LLamaContextParams.cs View File

@@ -22,7 +22,7 @@ namespace LLama.Native
public uint seed;

/// <summary>
/// text context
/// text context, 0 = from model
/// </summary>
public uint n_ctx;



+ 114
- 43
LLama/Native/NativeApi.cs View File

@@ -1,8 +1,13 @@
using System;
using System.Buffers;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using LLama.Exceptions;
using ManagedCuda;
#if NET6_0_OR_GREATER
using System.Runtime.Intrinsics.X86;
#endif

#pragma warning disable IDE1006 // Naming Styles

@@ -24,8 +29,9 @@ namespace LLama.Native
{
static NativeApi()
{
// Try to load a preferred library, based on CPU feature detection
TryLoadLibrary();
#if NET6_0_OR_GREATER
NativeLibrary.SetDllImportResolver(typeof(NativeApi).Assembly, LLamaImportResolver);
#endif

try
{
@@ -44,63 +50,120 @@ namespace LLama.Native
}

/// <summary>
/// Try to load libllama, using CPU feature detection to try and load a more specialised DLL if possible
/// Get the cuda version if possible.
/// </summary>
/// <returns>The library handle to unload later, or IntPtr.Zero if no library was loaded</returns>
private static IntPtr TryLoadLibrary()
/// <returns> -1 for no cuda</returns>
private static int GetCudaVersion()
{
#if NET6_0_OR_GREATER

if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
int deviceCount = CudaContext.GetDeviceCount();
for (int deviceIndex = 0; deviceIndex < deviceCount; deviceIndex++)
{
// All of the Windows libraries, in order of preference
return TryLoad("cu12.1.0/libllama.dll")
?? TryLoad("cu11.7.1/libllama.dll")
#if NET8_0_OR_GREATER
?? TryLoad("avx512/libllama.dll", System.Runtime.Intrinsics.X86.Avx512.IsSupported)
#endif
?? TryLoad("avx2/libllama.dll", System.Runtime.Intrinsics.X86.Avx2.IsSupported)
?? TryLoad("avx/libllama.dll", System.Runtime.Intrinsics.X86.Avx.IsSupported)
?? IntPtr.Zero;
using (CudaContext ctx = new CudaContext(deviceIndex))
{
var version = ctx.GetAPIVersionOfCurrentContext();
return version.Major;
}
}
return -1;
}

if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
// All of the Linux libraries, in order of preference
return TryLoad("cu12.1.0/libllama.so")
?? TryLoad("cu11.7.1/libllama.so")
/// <summary>
/// Get the xla flag for native library name.
/// </summary>
/// <returns></returns>
private static string GetAvxFlag()
{
AvxLevel level = AvxLevel.None;
#if NET6_0_OR_GREATER
if (Avx.IsSupported) level = AvxLevel.Avx;
if (Avx2.IsSupported) level = AvxLevel.Avx2;
#if NET8_0_OR_GREATER
?? TryLoad("avx512/libllama.so", System.Runtime.Intrinsics.X86.Avx512.IsSupported)
if(Avx512F.IsSupported) level = AvxLevel.Avx512;
#endif
?? TryLoad("avx2/libllama.so", System.Runtime.Intrinsics.X86.Avx2.IsSupported)
?? TryLoad("avx/libllama.so", System.Runtime.Intrinsics.X86.Avx.IsSupported)
?? IntPtr.Zero;
}

if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
return level switch
{
return TryLoad("runtimes/macos-arm64/libllama.dylib", System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported)
?? TryLoad("runtimes/macos-x86_64/libllama.dylib")
?? IntPtr.Zero;
}
AvxLevel.None => "",
AvxLevel.Avx => "-avx",
AvxLevel.Avx2 => "-avx2",
AvxLevel.Avx512 => "-avx512",
};
#else
return string.Empty;
#endif

return IntPtr.Zero;
}

#if NET6_0_OR_GREATER
// Try to load a DLL from the path if supported. Returns null if nothing is loaded.
static IntPtr? TryLoad(string path, bool supported = true)
private static IntPtr LLamaImportResolver(string name, Assembly assembly, DllImportSearchPath? searchPath)
{
IntPtr handle = IntPtr.Zero;
if(!name.Equals(libraryName))
{
if (!supported)
return null;

if (NativeLibrary.TryLoad(path, out var handle))
return handle;
return NativeLibrary.Load(name, assembly, searchPath);
}

return null;
string libraryPath = string.Empty;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var avxFlag = GetAvxFlag();
// check cuda
var cudaVersion = GetCudaVersion();
if(cudaVersion == 11)
{
libraryPath = $"runtimes/win-x64/native/libllama-cuda11{avxFlag}.dll";
}
else if (cudaVersion == 12)
{
libraryPath = $"runtimes/win-x64/native/libllama-cuda12{avxFlag}.dll";
}
else if(cudaVersion == -1) // cpu version
{
libraryPath = $"runtimes/win-x64/native/libllama{avxFlag}.dll";
}
else
{
throw new NotImplementedException($"Cuda version {cudaVersion} has not been supported, please compile dll yourself or open an issue in LLamaSharp.");
}
}
#endif
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
var avxFlag = GetAvxFlag();
// check cuda
var cudaVersion = GetCudaVersion();
if (cudaVersion == 11)
{
libraryPath = $"runtimes/linux-x64/native/libllama-cuda11{avxFlag}.so";
}
else if (cudaVersion == 12)
{
libraryPath = $"runtimes/linux-x64/native/libllama-cuda12{avxFlag}.so";
}
else if (cudaVersion == -1) // cpu version
{
libraryPath = $"runtimes/linux-x64/native/libllama{avxFlag}.so";
}
else
{
throw new NotImplementedException($"Cuda version {cudaVersion} has not been supported, please compile dll yourself or open an issue in LLamaSharp.");
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
if (System.Runtime.Intrinsics.Arm.ArmBase.Arm64.IsSupported)
{
libraryPath = $"runtimes/osx-arm64/native/libllama.dylib";
}
else
{
libraryPath = $"runtimes/osx-x64/native/libllama.dylib";
}
}

NativeLibrary.TryLoad(libraryPath, assembly, searchPath, out handle);
return handle;
}
#endif


private const string libraryName = "libllama";

@@ -574,5 +637,13 @@ namespace LLama.Native
/// <returns></returns>
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
public static extern int llama_set_n_threads(SafeLLamaContextHandle ctx, uint n_threads, uint n_threads_batch);

private enum AvxLevel
{
None = 0,
Avx = 1,
Avx2 = 2,
Avx512 = 3
}
}
}

+ 1
- 1
README.md View File

@@ -11,7 +11,7 @@


**The C#/.NET binding of [llama.cpp](https://github.com/ggerganov/llama.cpp). It provides higher-level APIs to inference the LLaMA Models and deploy it on local device with C#/.NET. It works on
both Windows, Linux and MAC without requirment for compiling llama.cpp yourself. Even without GPU or not enought GPU memory, you can still apply LLaMA models well with this repo. 🤗**
both Windows, Linux and MAC without requirment for compiling llama.cpp yourself. Even without GPU or not enough GPU memory, you can still apply LLaMA models well with this repo. 🤗**

**Furthermore, it provides integrations with other projects such as [semantic-kernel](https://github.com/microsoft/semantic-kernel), [kernel-memory](https://github.com/microsoft/kernel-memory) and [BotSharp](https://github.com/SciSharp/BotSharp) to provide higher-level applications.**



Write Preview
Loading…
Cancel
Save