From 5a79edeb51141ada9840c40b87a530d7323e08d1 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Wed, 10 May 2023 02:13:41 +0800 Subject: [PATCH] feat: add the framework and basic usages. --- .gitignore | 341 +++++++++ LLama.Console/LLama.Console.csproj | 15 + LLama.Console/Program.cs | 16 + LLama.Unittest/BasicTest.cs | 15 + LLama.Unittest/LLama.Unittest.csproj | 31 + LLama.Unittest/Usings.cs | 1 + LLama/Exceptions/RuntimeError.cs | 19 + LLama/LLamaCache.cs | 99 +++ LLama/LLamaModel.cs | 925 +++++++++++++++++++++++++ LLama/LLamaSharp.csproj | 23 + LLama/LLamaState.cs | 10 + LLama/LLamaTypes.cs | 34 + LLama/Logger.cs | 70 ++ LLama/Native/LLamaContextParams.cs | 65 ++ LLama/Native/LLamaFtype.cs | 20 + LLama/Native/LLamaTokenData.cs | 31 + LLama/Native/LLamaTokenDataArray.cs | 32 + LLama/Native/NativeApi.Sampling.cs | 128 ++++ LLama/Native/NativeApi.cs | 226 ++++++ LLama/Native/NativeInfo.cs | 15 + LLama/Native/SafeLLamaContextHandle.cs | 26 + LLama/Native/SafeLLamaHandleBase.cs | 32 + LLama/Native/SamplingApi.cs | 212 ++++++ LLamaSharp.sln | 51 ++ 24 files changed, 2437 insertions(+) create mode 100644 .gitignore create mode 100644 LLama.Console/LLama.Console.csproj create mode 100644 LLama.Console/Program.cs create mode 100644 LLama.Unittest/BasicTest.cs create mode 100644 LLama.Unittest/LLama.Unittest.csproj create mode 100644 LLama.Unittest/Usings.cs create mode 100644 LLama/Exceptions/RuntimeError.cs create mode 100644 LLama/LLamaCache.cs create mode 100644 LLama/LLamaModel.cs create mode 100644 LLama/LLamaSharp.csproj create mode 100644 LLama/LLamaState.cs create mode 100644 LLama/LLamaTypes.cs create mode 100644 LLama/Logger.cs create mode 100644 LLama/Native/LLamaContextParams.cs create mode 100644 LLama/Native/LLamaFtype.cs create mode 100644 LLama/Native/LLamaTokenData.cs create mode 100644 LLama/Native/LLamaTokenDataArray.cs create mode 100644 LLama/Native/NativeApi.Sampling.cs create mode 100644 LLama/Native/NativeApi.cs create mode 100644 LLama/Native/NativeInfo.cs create mode 100644 LLama/Native/SafeLLamaContextHandle.cs create mode 100644 LLama/Native/SafeLLamaHandleBase.cs create mode 100644 LLama/Native/SamplingApi.cs create mode 100644 LLamaSharp.sln diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..231d8379 --- /dev/null +++ b/.gitignore @@ -0,0 +1,341 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_i.h +*.ilk +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ +/docs/build +src/TensorFlowNET.Native/bazel-* +src/TensorFlowNET.Native/c_api.h +/.vscode +test/TensorFlowNET.Examples/mnist + + +# training model resources +.resources +/redist +*.xml +*.xsd diff --git a/LLama.Console/LLama.Console.csproj b/LLama.Console/LLama.Console.csproj new file mode 100644 index 00000000..7e98ba43 --- /dev/null +++ b/LLama.Console/LLama.Console.csproj @@ -0,0 +1,15 @@ + + + + Exe + net6.0 + enable + enable + AnyCPU;x64 + + + + + + + diff --git a/LLama.Console/Program.cs b/LLama.Console/Program.cs new file mode 100644 index 00000000..35a1c59b --- /dev/null +++ b/LLama.Console/Program.cs @@ -0,0 +1,16 @@ +using LLama; +using LLama.Types; + +string modelPath = @"D:\development\llama\weights\LLaMA\7B\ggml-model-q4_0.bin"; +LLamaModel model = new(modelPath, logits_all: false, verbose: false); +List chats = new List(); +chats.Add(new ChatCompletionMessage("user", "Hi, Alice, I'm Rinne.", null)); +chats.Add(new ChatCompletionMessage("assistant", "Hi, Rinne, I'm Alice. What can I do for you?", null)); +while (true) +{ + Console.Write("You: "); + var question = Console.ReadLine(); + chats.Add(new ChatCompletionMessage("user", question, null)); + var output = model.CreateChatCompletion(chats, max_tokens: 256); + Console.WriteLine($"LLama AI: {output.Choices[0].Message.Content}"); +} \ No newline at end of file diff --git a/LLama.Unittest/BasicTest.cs b/LLama.Unittest/BasicTest.cs new file mode 100644 index 00000000..641e3b06 --- /dev/null +++ b/LLama.Unittest/BasicTest.cs @@ -0,0 +1,15 @@ +namespace LLama.Unittest +{ + public class BasicTest + { + [Fact] + public void SimpleQA() + { + string modelPath = @"D:\development\llama\weights\LLaMA\7B\ggml-model-f32.bin"; + LLamaModel model = new(modelPath, logits_all: false); + var output = model.Call("Q: Why God makes many people believe him? A: ", max_tokens: 64, stop: new[] { "Q:", "\n" }, + echo: true); + Console.WriteLine(output); + } + } +} \ No newline at end of file diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj new file mode 100644 index 00000000..af96081a --- /dev/null +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -0,0 +1,31 @@ + + + + net6.0 + LLama.Unittest + enable + AnyCPU;x64 + enable + + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/LLama.Unittest/Usings.cs b/LLama.Unittest/Usings.cs new file mode 100644 index 00000000..8c927eb7 --- /dev/null +++ b/LLama.Unittest/Usings.cs @@ -0,0 +1 @@ +global using Xunit; \ No newline at end of file diff --git a/LLama/Exceptions/RuntimeError.cs b/LLama/Exceptions/RuntimeError.cs new file mode 100644 index 00000000..4d042d1d --- /dev/null +++ b/LLama/Exceptions/RuntimeError.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Exceptions +{ + public class RuntimeError: Exception + { + public RuntimeError() + { + + } + + public RuntimeError(string message): base(message) + { + + } + } +} diff --git a/LLama/LLamaCache.cs b/LLama/LLamaCache.cs new file mode 100644 index 00000000..94c44860 --- /dev/null +++ b/LLama/LLamaCache.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace LLama +{ + using llama_token = Int32; + /// + /// Cache for a llama.cpp model. + /// + public class LLamaCache + { + private Dictionary>> _cacheState; + private LinkedList> _cacheList; + private int _capacity; + + public int CacheSize + { + get + { + return _cacheState.Values.Select(s => s.Value.Value.Size).Sum(); + } + } + + /// + /// + /// + /// The max capacity (bytes). + public LLamaCache(int capacity = 2 << 30) + { + _cacheState = new(); + _cacheList = new(); + _capacity = capacity; + } + + public LLamaState this[llama_token[] key] + { + get + { + var prefixKey = FindLongestPrefixKey(key); + if(prefixKey is null) + { + throw new KeyNotFoundException(); + } + var value = _cacheState[prefixKey]; + MoveNodeToEnd(prefixKey); + return value.Value.Value; + } + set + { + var node = _cacheList.AddLast(new KeyValuePair(key, value)); + _cacheState[key] = node; + while(CacheSize > _capacity && _cacheList.Count > 0) + { + var topop = _cacheList.First; + _cacheState.Remove(topop.Value.Key); + _cacheList.RemoveFirst(); + } + } + } + + public bool Contains(llama_token[] key) + { + return FindLongestPrefixKey(key) is not null; + } + + private llama_token[]? FindLongestPrefixKey(llama_token[] key) + { + int minLen = 0; + llama_token[]? minKey = null; + var keys = _cacheState.Keys.Select(k => (k, LLamaModel.LongestTokenPrefix(k, key))); + foreach(var (k, prefixLen) in keys) + { + if(prefixLen > minLen) + { + minLen = prefixLen; + minKey = k; + } + } + return minKey; + } + + private void MoveNodeToEnd(llama_token[] key) + { + if (!_cacheState.TryGetValue(key, out var node)) + { + return; + } + + _cacheState.Remove(key); + _cacheList.Remove(node); + + var newNode = _cacheList.AddLast(new KeyValuePair(key, node.Value.Value)); + _cacheState.Add(key, newNode); + } + } +} diff --git a/LLama/LLamaModel.cs b/LLama/LLamaModel.cs new file mode 100644 index 00000000..08490509 --- /dev/null +++ b/LLama/LLamaModel.cs @@ -0,0 +1,925 @@ +using LLama.Exceptions; +using LLama.Native; +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using LLama.Types; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +namespace LLama +{ + using llama_token = Int32; + /// + /// High-level Wrapper of a llama.cpp model for inference. + /// + public class LLamaModel + { + private string _model_path; + LLamaContextParams _params; + private int _n_threads; + private int _n_batch; + private int _last_n_tokens_size; + private string? _lora_base; + private string? _lora_path; + private bool _verbose; + + private Queue _eval_tokens; + private Queue _eval_logits; + private LLamaCache? _cache; + private SafeLLamaContextHandle _ctx; + + private static readonly (int, int)[] _numAndPatterns = new (int, int)[] { (2, 192), (3, 224), (4, 240) }; + + /// + /// Load a llama.cpp model from the path. + /// + /// Note that the API is still unstable. The order of them is likely to + /// be changed in the future. It's recommened to specify the parameter name when + /// building your app. We use the cpp style parameter names here because it introduces + /// convenience for searching the docs. + /// Path to the model. + /// Maximum context size. + /// Number of parts to split the model into. If -1, the number of parts is automatically determined. + /// Random seed. 0 for random. + /// Use half-precision for key/value cache. + /// Return logits for all tokens, not just the last token. + /// Only load the vocabulary no weights. + /// Use mmap if possible. + /// Force the system to keep the model in RAM. + /// Embedding mode only. + /// Number of threads to use. If is not specified, the number of threads is automatically determined. + /// Maximum number of prompt tokens to batch together when calling llama_eval. + /// Maximum number of tokens to keep in the last_n_tokens deque. + /// Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. + /// Path to a LoRA file to apply to the model. + /// Print verbose output to stderr. + public LLamaModel(string model_path, int n_ctx = 512, int n_parts = -1, int seed = 1337, + bool f16_kv = true, bool logits_all = false, bool vocab_only = false, bool use_mmap = true, + bool use_mlock = false, bool embedding = false, int n_threads = -1, int n_batch = 512, + int last_n_tokens_size = 64, string? lora_base = null, string? lora_path = null, bool verbose = true) + { + _verbose = verbose; + _model_path = model_path; + + _params = NativeApi.llama_context_default_params(); + _params.n_ctx = n_ctx; + _params.n_parts = n_parts; + _params.seed = seed; + _params.f16_kv = f16_kv; + _params.logits_all = logits_all; + _params.vocab_only = vocab_only; + _params.use_mmap = lora_path is null ? use_mmap : false; + _params.use_mlock = use_mlock; + _params.embedding = embedding; + + _last_n_tokens_size = last_n_tokens_size; + _n_batch = Math.Min(n_ctx, n_batch); + + _eval_tokens = new Queue(capacity: n_ctx); + _eval_logits = new Queue(logits_all ? n_ctx : 1); + + _cache = null; + + _n_threads = n_threads; + if(_n_threads == -1) + { + _n_threads = Math.Max(Environment.ProcessorCount / 2, 1); + } + + _lora_base = lora_base; + _lora_path = lora_path; + + if(!File.Exists(model_path) && !Directory.Exists(model_path)) + { + throw new FileNotFoundException($"Model path does not exist: {model_path}"); + } + + // Move from heap to stack to prevent the moving. + _ctx = new SafeLLamaContextHandle(NativeApi.llama_init_from_file(Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(model_path)), _params)); + + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + + if(_lora_path is not null) + { + if(NativeApi.llama_apply_lora_from_file(_ctx, lora_path, lora_base, _n_threads) != 0) + { + throw new RuntimeError($"Failed to apply LoRA from lora path: {_lora_path} to base path: {_lora_base}"); + } + } + + if (_verbose) + { +#if NET6_0_OR_GREATER + Logger.Default.Info(Marshal.PtrToStringUTF8(NativeApi.llama_print_system_info())); +#endif + } + } + + public LLamaModel(LLamaModel other) + { + _ctx = other._ctx; + _model_path = other._model_path; + _params = other._params; + _last_n_tokens_size = other._last_n_tokens_size; + _n_threads = other._n_threads; + _n_batch = other._n_batch; + _verbose = other._verbose; + _lora_base = other._lora_base; + _lora_path = other._lora_path; + _eval_logits = new Queue(other._eval_logits); + _eval_tokens = new Queue(other._eval_tokens); + } + + /// + /// Tokenize a string. + /// + /// The utf-8 encoded string to tokenize. + /// A list of tokens. + /// If the tokenization failed. + public List Tokenize(string text) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var n_ctx = NativeApi.llama_n_ctx(_ctx); + var tokens = new llama_token[n_ctx]; + var n_tokens = NativeApi.llama_tokenize(_ctx, text, tokens, n_ctx, true); + if(n_tokens < 0) + { + throw new RuntimeError($"Failed to tokenize: text=\"{text}\" n_tokens={n_tokens}"); + } + return tokens.Take(n_tokens).ToList(); + } + + /// + /// Detokenize a list of tokens. + /// + /// The list of tokens to detokenize. + /// The detokenized string. + public string DeTokenize(IEnumerable tokens) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + string output = ""; + foreach(var token in tokens) + { +#if NET6_0_OR_GREATER + output += Marshal.PtrToStringUTF8(NativeApi.llama_token_to_str(_ctx, token)); +#else + output += Marshal.PtrToStringAnsi(NativeApi.llama_token_to_str(_ctx, token)); +#endif + } + return output; + } + + /// + /// Set the cache. + /// + /// The cache to set. + public void SetCache(LLamaCache? cache) + { + _cache = cache; + } + + /// + /// Reset the model state. + /// + public void Reset() + { + _eval_tokens.Clear(); + _eval_logits.Clear(); + } + + /// + /// Evaluate a list of tokens. + /// + /// The list of tokens to evaluate. + /// + public unsafe void Eval(List tokens) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var n_ctx = NativeApi.llama_n_ctx(_ctx); + for(int i = 0; i < tokens.Count; i += _n_batch) + { + var batch = tokens.Take(Math.Min(tokens.Count, i + _n_batch)).Skip(i); + llama_token n_past = Math.Min(n_ctx - batch.Count(), _eval_tokens.Count); + llama_token n_tokens = batch.Count(); + llama_token return_code = NativeApi.llama_eval( + ctx: _ctx, + tokens: batch.ToArray(), + n_tokens: n_tokens, + n_past: n_past, + n_threads: _n_threads + ); + if(return_code != 0) + { + throw new RuntimeError($"llama_eval returned {return_code}"); + } + foreach(var b in batch) + { + _eval_tokens.Enqueue(b); + } + int rows = _params.logits_all ? n_tokens : 1; + llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); + var cols = n_vocab; + var logits_view = NativeApi.llama_get_logits(_ctx); + for(int j = 0; j < rows; j++) + { + float[] logit = new float[cols]; + for(int k = 0; k < cols; k++) + { + logit[k] = logits_view[j * cols + k]; + } + _eval_logits.Enqueue(logit); + } + } + } + + private llama_token SampleInternal(llama_token[] last_n_tokens_data, int last_n_tokens_size, int top_k, + float top_p, float temp, float repeat_penalty, float frequency_penalty, float presence_penalty) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + Debug.Assert(_eval_logits.Count > 0); + llama_token n_vocab = NativeApi.llama_n_vocab(_ctx); + var logits = _eval_logits.Last(); + LLamaTokenData[] data = new LLamaTokenData[n_vocab]; + for(int i = 0; i < n_vocab; i++) + { + data[i] = new LLamaTokenData(i, logits[i], .0f); + } + ulong size = (ulong)n_vocab; + bool sorted = false; + LLamaTokenDataArray candidates = new(data, size, sorted); + SamplingApi.llama_sample_repetition_penalty(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, + repeat_penalty); + //SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates, last_n_tokens_data, (ulong)last_n_tokens_size, + // frequency_penalty, presence_penalty); + if(temp == .0f) + { + return SamplingApi.llama_sample_token_greedy(_ctx, candidates); + } + else + { + SamplingApi.llama_sample_top_k(_ctx, candidates, top_k, 1); + SamplingApi.llama_sample_tail_free(_ctx, candidates, 1.0f, 1); + SamplingApi.llama_sample_typical(_ctx, candidates, 1.0f, 1); + SamplingApi.llama_sample_top_p(_ctx, candidates, top_p, 1); + SamplingApi.llama_sample_temperature(_ctx, candidates, temp); + return SamplingApi.llama_sample_token(_ctx, candidates); + } + } + + /// + /// Sample a token from the model. + /// + /// The top-k sampling parameter. + /// The top-p sampling parameter. + /// The temperature parameter. + /// The repeat penalty parameter. + /// + /// + /// The sampled token. + public llama_token Sample(int top_k, float top_p, float temp, float repeat_penalty, float frequency_penalty = .0f, + float presence_penalty = .0f) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + var last_n_tokens_data = Enumerable.Repeat(0, Math.Max(0, _last_n_tokens_size - _eval_tokens.Count)); + last_n_tokens_data = last_n_tokens_data.Concat(_eval_tokens.ToList() + .Skip(Math.Max(0, _eval_tokens.Count - _last_n_tokens_size))); + llama_token[] tokens_data = new llama_token[_last_n_tokens_size]; + int i = 0; + foreach(var data in last_n_tokens_data) + { + if(i < _last_n_tokens_size) + { + tokens_data[i++] = data; + } + else + { + break; + } + } + return SampleInternal(tokens_data, _last_n_tokens_size, top_k, top_p, temp, repeat_penalty, frequency_penalty, presence_penalty); + } + + /// + /// Create a generator of tokens from a prompt. + /// + /// + /// Examples: + /// var llama = new LlamaModel("models/ggml-7b.bin") + /// var tokens = llama.Tokenize(b"Hello, world!") + /// foreach(var token in llama.Generate(tokens, top_k:40, top_p:0.95, temp:1.0, repeat_penalty:1.1)){ + /// Console.WriteLine(llama.DeTokenize(new []{token})); + /// } + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public IEnumerable Generate(IEnumerable tokens, int top_k, float top_p, float temp, + float repeat_penalty, float frequency_penalty = .0f, float presence_penalty = .0f, bool reset = true) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + if(reset && _eval_tokens.Count > 0) + { + int longest_prefix = 0; + foreach(var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count() - 1), (x, y) => (x, y))) + { + if(a == b) + { + longest_prefix += 1; + } + else + { + break; + } + } + if(longest_prefix > 0) + { + if (_verbose) + { + Logger.Default.Info("Llama.generate: prefix-match hit"); + } + reset = false; + tokens = tokens.Skip(longest_prefix); + for(int i = 0; i < _eval_tokens.Count - longest_prefix; i++) + { + _eval_tokens.Dequeue(); + if(_eval_logits.Count > 0) + { + _eval_logits.Dequeue(); + } + } + } + } + + if (reset) + { + Reset(); + } + + while (true) + { + Eval(tokens.ToList()); + var token = Sample(top_k, top_p, temp, frequency_penalty, presence_penalty, repeat_penalty); + yield return token; + // TODO(Rinne): verify if the implementation is correct. + } + } + + /// + /// Embed a string. + /// + /// The utf-8 encoded string to embed. + /// An embedding object. + /// + public unsafe Embedding CreateEmbedding(string input) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + if (!_params.embedding) + { + throw new RuntimeError("Llama model must be created with embedding=True to call this method"); + } + + if (_verbose) + { + NativeApi.llama_reset_timings(_ctx); + } + + var tokens = Tokenize(input); + Reset(); + Eval(tokens); + int n_tokens = tokens.Count; + var embeddingPtr = NativeApi.llama_get_embeddings(_ctx); + int cnt = NativeApi.llama_n_embd(_ctx); + float[] embedding = new float[cnt]; + for(int i = 0; i < cnt; i++) + { + embedding[i] = embeddingPtr[i]; + } + + if (_verbose) + { + NativeApi.llama_print_timings(_ctx); + } + + return new Embedding("list", _model_path, new[] { new EmbeddingData(0, "embedding", embedding) }, + new EmbeddingUsage(n_tokens, n_tokens)); + } + + public float[] Embed(string input) + { + return CreateEmbedding(input).Data[0].Embedding; + } + + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// IEnumerable of Completion and CompletionChunk + /// + private IEnumerable CreateCompletionInternal(string prompt, string?suffix = null, int max_tokens = 16, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40, bool stream = false) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + string completionId = $"cmpl-{Guid.NewGuid()}"; + var created = DateTime.Now.Millisecond; + List completionTokens = new List(); + + var promptTokens = Tokenize($" {prompt}"); + string text = ""; + int returnedCharacters = 0; + if(stop is null) + { + stop = new string[0]; + } + + if (_verbose) + { + NativeApi.llama_reset_timings(_ctx); + } + + if(promptTokens.Count + max_tokens > NativeApi.llama_n_ctx(_ctx)) + { + throw new ArgumentException($"Requested tokens exceed context window of {NativeApi.llama_n_ctx(_ctx)}"); + } + + if(logprobs != -1 && !_params.logits_all) + { + throw new ArgumentException("logprobs is not supported for models created with logits_all=False"); + } + + if(_cache is not null) + { + try + { + // TODO(Rinne): revise it since it will compare reference instead of elements. + var cacheItem = _cache[promptTokens.ToArray()]; + var cachePrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); + var evalPrefixLen = LongestTokenPrefix(_eval_tokens.AsEnumerable(), promptTokens); + if(cachePrefixLen > evalPrefixLen) + { + LoadState(cacheItem); + if (_verbose) + { + Logger.Default.Info("Llama._create_completion: cache hit"); + } + } + } + catch (KeyNotFoundException) + { + if (_verbose) + { + Logger.Default.Warn("Llama._create_completion: cache miss"); + } + } + } + + string finishReason = "length"; + int multibyteFix = 0; + bool reset = true; + List tokens = new(promptTokens); + if (reset && _eval_tokens.Count > 0) + { + int longest_prefix = 0; + foreach (var (a, b) in _eval_tokens.ToList().Zip(tokens.Take(tokens.Count - 1), (x, y) => (x, y))) + { + if (a == b) + { + longest_prefix += 1; + } + else + { + break; + } + } + if (longest_prefix > 0) + { + if (_verbose) + { + Logger.Default.Info("Llama.generate: prefix-match hit"); + } + reset = false; + tokens = tokens.Skip(longest_prefix).ToList(); + for (int i = 0; i < _eval_tokens.Count - longest_prefix; i++) + { + _eval_tokens.Dequeue(); + if (_eval_logits.Count > 0) + { + _eval_logits.Dequeue(); + } + } + } + } + + if (reset) + { + Reset(); + } + //foreach (var token in Generate(promptTokens, top_k, top_p, temperature, frequency_penalty, presence_penalty, repeat_penalty)) + while(true) + { + Eval(tokens); + var token = Sample(top_k, top_p, temperature, repeat_penalty, frequency_penalty, presence_penalty); + tokens.Clear(); + tokens.Add(token); + if (token == NativeApi.llama_token_eos()) + { + text = DeTokenize(completionTokens); + finishReason = "stop"; + break; + } + + completionTokens.Add(token); + + string allText = DeTokenize(completionTokens); + + int cut = Math.Min(3, allText.Length); + for(int i = allText.Length - cut; i < allText.Length; i++) + { + var c = (int)allText[i]; + int k = cut - i; + foreach(var (num, pattern) in _numAndPatterns) + { + if(num > k && (pattern & c) == pattern) + { + multibyteFix = num - k; + } + } + } + + if(multibyteFix > 0) + { + multibyteFix--; + continue; + } + + var anyStop = stop.Where(s => allText.Contains(s)); + if(anyStop.Count() > 0) + { + var firstStop = anyStop.First(); + text = allText.Substring(0, allText.IndexOf(firstStop)); + finishReason = "stop"; + break; + } + + if (stream) + { + var start = returnedCharacters; + int longest = 0; + foreach(var s in stop) + { + for(int i = s.Length; i > 0; i--) + { + if(allText.EndsWith(s.Substring(0, i))) + { + if(i > longest) + { + longest = i; + } + break; + } + } + } + text = allText.Substring(0, allText.Length - longest); + returnedCharacters += text.Skip(start).Count(); + yield return new CompletionChunk(completionId, "text_completion", created, _model_path, new CompletionChoice[] + { + new CompletionChoice(text.Substring(returnedCharacters), 0, null, finishReason) + }); + } + } + + if(_cache is not null) + { + if (_verbose) + { + Logger.Default.Info("Llama._create_completion: cache save"); + } + _cache[promptTokens.Concat(completionTokens).ToArray()] = SaveState(); + } + + if (stream) + { + yield return new CompletionChunk(completionId, "text_completion", created, _model_path, new CompletionChoice[] + { + new CompletionChoice(text.Substring(returnedCharacters), 0, null, finishReason) + }); + } + + string textStr = text; + if (echo) + { + textStr = prompt + textStr; + } + if(suffix is not null) + { + textStr = textStr + suffix; + } + + CompletionLogprobs? logProbs = null; + if (logprobs != -1) + { + int textOffset = 0; + List textOffsets = new(); + List tokenLogprobs = new(); + List tokenStrs = new(); + List> topLogprobs = new(); + + var allTokens = promptTokens.Concat(completionTokens).ToArray(); + var allTokenStrs = allTokens.Select(t => DeTokenize(new[] { t })); + var allLogProbs = _eval_logits.Select(row => LogitsToLogprobs(row)); + + foreach (var (token, tokenStr, logProbsToken) in allTokens.Zip(allTokenStrs, (x, y) => (x, y)) + .Zip(allLogProbs, (x, y) => (x.x, x.y, y))) + { + textOffsets.Add(textOffset); + textOffset += tokenStr.Length; + tokenStrs.Add(tokenStr); + var sortedLogprobs = logProbsToken.Zip(Enumerable.Range(0, logProbsToken.Count()), (x, y) => (x, y)) + .OrderByDescending(x => x.x).ToList(); + tokenLogprobs.Add(sortedLogprobs[token].x); + var topLogprob = sortedLogprobs.Take(logprobs).ToDictionary(t => DeTokenize(new[] { t.y }), t => t.x); + topLogprob[tokenStr] = sortedLogprobs[token].x; + topLogprobs.Add(topLogprob); + } + + logProbs = new(textOffsets.ToArray(), tokenLogprobs.ToArray(), tokenStrs.ToArray(), topLogprobs.ToArray()); + } + + if (_verbose) + { + NativeApi.llama_print_timings(_ctx); + } + + yield return new Completion(completionId, "text_completion", created, _model_path, new CompletionChoice[] + { + new CompletionChoice(text, 0, logProbs, finishReason) + }, new CompletionUsage(promptTokens.Count, completionTokens.Count, promptTokens.Count + completionTokens.Count)); + } + + /// + /// Generate text from a prompt and yield return the result. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public IEnumerable CreateCompletionStream(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + yield return (CompletionChunk)CreateCompletionInternal(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k, true); + } + + /// + /// Generate text from a prompt. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public Completion CreateCompletion(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + var completion = CreateCompletionInternal(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k, false).First(); + return (Completion)completion; + } + + /// + /// Generate text from a prompt. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public Completion Call(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + return CreateCompletion(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k); + } + + /// + /// Generate text from a prompt and yield return the result. + /// + /// The prompt to generate text from. + /// A suffix to append to the generated text. If None, no suffix is appended. + /// The maximum number of tokens to generate. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The number of logprobs to return. If None, no logprobs are returned. + /// Whether to echo the prompt. + /// A list of strings to stop generation when encountered. + /// + /// + /// The penalty to apply to repeated tokens. + /// The top-k value to use for sampling. + /// + public IEnumerable StreamCall(string prompt, string? suffix = null, int max_tokens = 128, float temperature = 0.8f, + float top_p = 0.95f, int logprobs = -1, bool echo = false, string[]? stop = null, float frequency_penalty = .0f, + float presence_penalty = .0f, float repeat_penalty = 1.1f, int top_k = 40) + { + return CreateCompletionStream(prompt, suffix, max_tokens, temperature, top_p, logprobs, echo, stop, + frequency_penalty, presence_penalty, repeat_penalty, top_k); + } + + private ChatCompletion ConvertTextCompletionToChat(Completion completion) + { + return new ChatCompletion($"chat{completion.Id}", "chat.completion", completion.Created, completion.Model, + new[] { new ChatCompletionChoice(0, new ChatCompletionMessage("assistant", completion.Choices[0].Text, null), + completion.Choices[0].FinishReason) }, completion.Usage); + } + + private IEnumerable ConvertTextCompletionChunksToChat(IEnumerable chunks) + { + bool isFirst = true; + foreach(var chunk in chunks) + { + if(isFirst) + { + yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, + new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta("assistant", null), null) }); + isFirst = false; + } + yield return new ChatCompletionChunk($"chat{chunk.Id}", chunk.Model, "chat.completion.chunk", chunk.Created, + new[] { new ChatCompletionChunkChoice(0, new ChatCompletionChunkDelta(null, chunk.Choices[0].Text), + chunk.Choices[0].FinishReason) }); + } + } + + /// + /// Generate a chat completion from a list of messages. + /// + /// A list of messages to generate a response for. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The top-k value to use for sampling. + /// A list of strings to stop generation when encountered. + /// The maximum number of tokens to generate. + /// + /// + /// The penalty to apply to repeated tokens. + /// + public ChatCompletion CreateChatCompletion(IEnumerable messages, float temperature = .2f, float top_p = .95f, + int top_k = 40, string[]? stop = null, int max_tokens = 256, float presence_penalty = .0f, float frequency_penalty = .0f, + float repeat_penalty = 1.1f) + { + if(stop is null) + { + stop = new string[0]; + } + string GetRole(ChatCompletionMessage message) + { + return message.Role == "user" ? "Human" : "Assistant"; + } + string chatHistory = string.Join("", messages.Select(m => $"### {GetRole(m)}:{m.Content}")); + var prompt = chatHistory + "### Assistant:"; + var promptStop = new[] { "### Assistant:", "### Human:" }.Concat(stop).ToArray(); + var completion = Call(prompt, stop: promptStop, temperature: temperature, top_p: top_p, top_k: top_k, max_tokens: max_tokens, + repeat_penalty: repeat_penalty, presence_penalty: presence_penalty, frequency_penalty: frequency_penalty); + return ConvertTextCompletionToChat(completion); + } + + /// + /// Generate a chat completion from a list of messages and yield return the result. + /// + /// A list of messages to generate a response for. + /// The temperature to use for sampling. + /// The top-p value to use for sampling. + /// The top-k value to use for sampling. + /// A list of strings to stop generation when encountered. + /// The maximum number of tokens to generate. + /// + /// + /// The penalty to apply to repeated tokens. + /// + public IEnumerable CreateChatCompletionStream(IEnumerable messages, float temperature = .2f, float top_p = .95f, + int top_k = 40, string[]? stop = null, int max_tokens = 256, float presence_penalty = .0f, float frequency_penalty = .0f, + float repeat_penalty = 1.1f) + { + if (stop is null) + { + stop = new string[0]; + } + string GetRole(ChatCompletionMessage message) + { + return message.Role == "user" ? "Human" : "Assistant"; + } + string chatHistory = string.Join("", messages.Select(m => $"### {GetRole(m)}:{m.Content}")); + var prompt = chatHistory + "### Assistant:"; + var promptStop = new[] { "### Assistant:", "### Human:" }.Concat(stop).ToArray(); + var completion = StreamCall(prompt, stop: promptStop, temperature: temperature, top_p: top_p, top_k: top_k, max_tokens: max_tokens, + repeat_penalty: repeat_penalty, presence_penalty: presence_penalty, frequency_penalty: frequency_penalty); + return ConvertTextCompletionChunksToChat(completion); + } + + public LLamaState SaveState() + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + ulong stateSize = NativeApi.llama_get_state_size(_ctx); + byte[] llamaState = new byte[stateSize]; + ulong nBytes = NativeApi.llama_copy_state_data(_ctx, llamaState); + if(nBytes > stateSize) + { + throw new RuntimeError("Failed to copy llama state data"); + } + byte[] llamaStateCompact = new byte[nBytes]; + llamaState.Take((int)nBytes).ToArray().CopyTo(llamaStateCompact, 0); + if (_verbose) + { + Logger.Default.Info($"Llama.save_state: saving {nBytes} bytes of llama state"); + } + return new LLamaState(new Queue(_eval_tokens), new Queue(_eval_logits), + llamaStateCompact, (int)nBytes); + } + + public void LoadState(LLamaState state) + { + Debug.Assert(_ctx.DangerousGetHandle() != IntPtr.Zero); + _eval_tokens = new Queue(state.EvalTokens); + _eval_logits = new Queue(state.EvalLogits); + if(NativeApi.llama_set_state_data(_ctx, state.State) != (ulong)state.Size) + { + throw new RuntimeError($"Failed to set llama state data"); + } + } + + private static IEnumerable LogitsToLogprobs(IEnumerable logits) + { + var exps = logits.Select(x => (float)Math.Exp(x)); + var sumExps = exps.Sum(); + return exps.Select(x => (float)Math.Log(x / sumExps)); + } + + internal static int LongestTokenPrefix(IEnumerable a, IEnumerable b) + { + int longestPrefix = 0; + foreach(var (x, y) in a.Zip(b, (x, y) => (x, y))) + { + if(x == y) + { + longestPrefix++; + } + else + { + break; + } + } + return longestPrefix; + } + } +} diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj new file mode 100644 index 00000000..fd26090a --- /dev/null +++ b/LLama/LLamaSharp.csproj @@ -0,0 +1,23 @@ + + + + netstandard2.0;net6.0 + LLama + enable + 10 + AnyCPU;x64 + True + + + + + + + + + + + + + + diff --git a/LLama/LLamaState.cs b/LLama/LLamaState.cs new file mode 100644 index 00000000..397d3431 --- /dev/null +++ b/LLama/LLamaState.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama +{ + using llama_token = Int32; + public record LLamaState(Queue EvalTokens, Queue EvalLogits, + byte[] State, int Size); +} diff --git a/LLama/LLamaTypes.cs b/LLama/LLamaTypes.cs new file mode 100644 index 00000000..3f21b9f7 --- /dev/null +++ b/LLama/LLamaTypes.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Types +{ + public record EmbeddingUsage(int PromptTokens, int TotalTokens); + + public record EmbeddingData(int Index, string Object, float[] Embedding); + + public record Embedding(string Object, string Model, EmbeddingData[] Data, EmbeddingUsage Usage); + + public record CompletionLogprobs(int[] TextOffset, float[] TokenLogProbs, string[] Tokens, Dictionary[] TopLogprobs); + + public record CompletionChoice(string Text, int Index, CompletionLogprobs? Logprobs, string? FinishReason); + + public record CompletionUsage(int PromptTokens, int CompletionTokens, int TotalTokens); + + public record CompletionChunk(string Id, string Object, int Created, string Model, CompletionChoice[] Choices); + + public record Completion(string Id, string Object, int Created, string Model, CompletionChoice[] Choices, CompletionUsage Usage); + + public record ChatCompletionMessage(string Role, string Content, string? User); + + public record ChatCompletionChoice(int Index, ChatCompletionMessage Message, string? FinishReason); + + public record ChatCompletion(string Id, string Object, int Created, string Model, ChatCompletionChoice[] Choices, CompletionUsage Usage); + + public record ChatCompletionChunkDelta(string? Role, string? Content); + + public record ChatCompletionChunkChoice(int Index, ChatCompletionChunkDelta Delta, string? FinishReason); + + public record ChatCompletionChunk(string Id, string Model, string Object, int Created, ChatCompletionChunkChoice[] Choices); +} diff --git a/LLama/Logger.cs b/LLama/Logger.cs new file mode 100644 index 00000000..00abbcbe --- /dev/null +++ b/LLama/Logger.cs @@ -0,0 +1,70 @@ +using System; +using Microsoft.Extensions.Logging; +using Serilog; +public sealed class Logger +{ + private static readonly Lazy _instance = new Lazy(() => new Logger()); + private static ILoggerFactory _loggerFactory; + private static readonly object _lock = new object(); + + public static Logger Default => _instance.Value; + + private Logger() + { + var logConfig = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Console(outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Message}{NewLine}{Exception}"); + + _loggerFactory = LoggerFactory.Create(builder => + { + builder.AddSerilog(logConfig.CreateLogger(), dispose: true); + }); + } + + public void ToConsole() + { + // 不需要处理，Serilog 默认就输出到控制台 + } + + public void ToFile(string filename) + { + var logConfig = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.Console(outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Message}{NewLine}{Exception}") + .WriteTo.File(filename, outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {Message}{NewLine}{Exception}"); + + lock (_lock) + { + _loggerFactory.Dispose(); + + _loggerFactory = LoggerFactory.Create(builder => + { + builder.AddSerilog(logConfig.CreateLogger(), dispose: true); + }); + } + } + + public void Info(string message) + { + _loggerFactory.CreateLogger().LogInformation(message); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(message); + Console.ResetColor(); + } + + public void Warn(string message) + { + _loggerFactory.CreateLogger().LogWarning(message); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(message); + Console.ResetColor(); + } + + public void Error(string message) + { + _loggerFactory.CreateLogger().LogError(message); + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine(message); + Console.ResetColor(); + } +} \ No newline at end of file diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs new file mode 100644 index 00000000..4e782cc9 --- /dev/null +++ b/LLama/Native/LLamaContextParams.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + public delegate void LlamaProgressCallback(float progress, IntPtr ctx); + [StructLayout(LayoutKind.Sequential)] + public struct LLamaContextParams + { + /// + /// text context + /// + public int n_ctx; + /// + /// -1 for default + /// + public int n_parts; + /// + /// RNG seed, -1 for random + /// + public int seed; + + /// + /// use fp16 for KV cache + /// + [MarshalAs(UnmanagedType.I1)] + public bool f16_kv; + /// + /// the llama_eval() call computes all logits, not just the last one + /// + [MarshalAs(UnmanagedType.I1)] + public bool logits_all; + /// + /// only load the vocabulary, no weights + /// + [MarshalAs(UnmanagedType.I1)] + public bool vocab_only; + /// + /// use mmap if possible + /// + [MarshalAs(UnmanagedType.I1)] + public bool use_mmap; + /// + /// force system to keep model in RAM + /// + [MarshalAs(UnmanagedType.I1)] + public bool use_mlock; + /// + /// embedding mode only + /// + [MarshalAs(UnmanagedType.I1)] + public bool embedding; + + /// + /// called with a progress value between 0 and 1, pass NULL to disable + /// + public IntPtr progress_callback; + /// + /// context pointer passed to the progress callback + /// + public IntPtr progress_callback_user_data; + } +} diff --git a/LLama/Native/LLamaFtype.cs b/LLama/Native/LLamaFtype.cs new file mode 100644 index 00000000..e196e90d --- /dev/null +++ b/LLama/Native/LLamaFtype.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Native +{ + internal enum LLamaFtype + { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_3 (6) support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + } +} diff --git a/LLama/Native/LLamaTokenData.cs b/LLama/Native/LLamaTokenData.cs new file mode 100644 index 00000000..cc877e80 --- /dev/null +++ b/LLama/Native/LLamaTokenData.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + [StructLayout(LayoutKind.Sequential)] + internal struct LLamaTokenData + { + /// + /// token id + /// + public int id; + /// + /// log-odds of the token + /// + public float logit; + /// + /// probability of the token + /// + public float p; + + public LLamaTokenData(int id, float logit, float p) + { + this.id = id; + this.logit = logit; + this.p = p; + } + } +} diff --git a/LLama/Native/LLamaTokenDataArray.cs b/LLama/Native/LLamaTokenDataArray.cs new file mode 100644 index 00000000..78d71e53 --- /dev/null +++ b/LLama/Native/LLamaTokenDataArray.cs @@ -0,0 +1,32 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + [StructLayout(LayoutKind.Sequential)] + internal struct LLamaTokenDataArray + { + public Memory data; + public ulong size; + [MarshalAs(UnmanagedType.I1)] + public bool sorted; + + public LLamaTokenDataArray(LLamaTokenData[] data, ulong size, bool sorted) + { + this.data = data; + this.size = size; + this.sorted = sorted; + } + } + + [StructLayout(LayoutKind.Sequential)] + internal struct LLamaTokenDataArrayNative + { + public IntPtr data; + public ulong size; + public bool sorted; + } +} diff --git a/LLama/Native/NativeApi.Sampling.cs b/LLama/Native/NativeApi.Sampling.cs new file mode 100644 index 00000000..87210cfb --- /dev/null +++ b/LLama/Native/NativeApi.Sampling.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + using llama_token = Int32; + internal partial class NativeApi + { + /// + /// Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_repetition_penalty(SafeLLamaContextHandle ctx, IntPtr candidates, llama_token[] last_tokens, ulong last_tokens_size, float penalty); + + /// + /// Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_frequency_and_presence_penalties(SafeLLamaContextHandle ctx, IntPtr candidates, llama_token[] last_tokens, ulong last_tokens_size, float alpha_frequency, float alpha_presence); + + /// + /// Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. + /// + /// + /// Pointer to LLamaTokenDataArray + [DllImport(libraryName)] + public static extern void llama_sample_softmax(SafeLLamaContextHandle ctx, IntPtr candidates); + + /// + /// Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_top_k(SafeLLamaContextHandle ctx, IntPtr candidates, int k, ulong min_keep); + + /// + /// Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_top_p(SafeLLamaContextHandle ctx, IntPtr candidates, float p, ulong min_keep); + + /// + /// Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_tail_free(SafeLLamaContextHandle ctx, IntPtr candidates, float z, ulong min_keep); + + /// + /// Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + [DllImport(libraryName)] + public static extern void llama_sample_typical(SafeLLamaContextHandle ctx, IntPtr candidates, float p, ulong min_keep); + + [DllImport(libraryName)] + public static extern void llama_sample_temperature(SafeLLamaContextHandle ctx, IntPtr candidates, float temp); + + /// + /// Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// + /// + /// A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + /// + [DllImport(libraryName)] + public static extern llama_token llama_sample_token_mirostat(SafeLLamaContextHandle ctx, IntPtr candidates, float tau, float eta, int m, float[] mu); + + /// + /// Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// + /// + /// A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + /// + [DllImport(libraryName)] + public static extern llama_token llama_sample_token_mirostat_v2(SafeLLamaContextHandle ctx, IntPtr candidates, float tau, float eta, float[] mu); + + /// + /// Selects the token with the highest probability. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + [DllImport(libraryName)] + public static extern llama_token llama_sample_token_greedy(SafeLLamaContextHandle ctx, IntPtr candidates); + + /// + /// Randomly selects a token from the candidates based on their probabilities. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + [DllImport(libraryName)] + public static extern llama_token llama_sample_token(SafeLLamaContextHandle ctx, IntPtr candidates); + } +} diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs new file mode 100644 index 00000000..4d55a639 --- /dev/null +++ b/LLama/Native/NativeApi.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + using llama_token = Int32; + internal unsafe partial class NativeApi + { + private const string libraryName = "llama"; + + [DllImport(libraryName)] + public static extern LLamaContextParams llama_context_default_params(); + + [DllImport(libraryName)] + public static extern bool llama_mmap_supported(); + + [DllImport(libraryName)] + public static extern bool llama_mlock_supported(); + + /// + /// Various functions for loading a ggml llama model. + /// Allocate (almost) all memory needed for the model. + /// Return NULL on failure + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern IntPtr llama_init_from_file(string path_model, LLamaContextParams params_); + + /// + /// Frees all allocated memory + /// + /// + [DllImport(libraryName)] + public static extern void llama_free(IntPtr ctx); + + /// + /// Returns 0 on success + /// + /// + /// + /// + /// how many threads to use. If <=0, will use std::thread::hardware_concurrency(), else the number given + /// not great API - very likely to change + /// Returns 0 on success + [DllImport(libraryName)] + public static extern int llama_model_quantize(string fname_inp, string fname_out, LLamaFtype ftype, int nthread); + + /// + /// Apply a LoRA adapter to a loaded model + /// path_base_model is the path to a higher quality model to use as a base for + /// the layers modified by the adapter. Can be NULL to use the current loaded model. + /// The model needs to be reloaded before applying a new adapter, otherwise the adapter + /// will be applied on top of the previous one + /// + /// + /// + /// + /// + /// Returns 0 on success + [DllImport(libraryName)] + public static extern int llama_apply_lora_from_file(SafeLLamaContextHandle ctx, string path_lora, string path_base_model, int n_threads); + + /// + /// Returns the number of tokens in the KV cache + /// + /// + /// + [DllImport(libraryName)] + public static extern int llama_get_kv_cache_token_count(SafeLLamaContextHandle ctx); + + /// + /// Sets the current rng seed. + /// + /// + /// + [DllImport(libraryName)] + public static extern void llama_set_rng_seed(SafeLLamaContextHandle ctx, int seed); + + /// + /// Returns the maximum size in bytes of the state (rng, logits, embedding + /// and kv_cache) - will often be smaller after compacting tokens + /// + /// + /// + [DllImport(libraryName)] + public static extern ulong llama_get_state_size(SafeLLamaContextHandle ctx); + + /// + /// Copies the state to the specified destination address. + /// Destination needs to have allocated enough memory. + /// Returns the number of bytes copied + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern ulong llama_copy_state_data(SafeLLamaContextHandle ctx, byte[] dest); + + /// + /// Set the state reading from the specified address + /// Returns the number of bytes read + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern ulong llama_set_state_data(SafeLLamaContextHandle ctx, byte[] src); + + /// + /// Load session file + /// + /// + /// + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern bool llama_load_session_file(SafeLLamaContextHandle ctx, string path_session, llama_token[] tokens_out, ulong n_token_capacity, ulong[] n_token_count_out); + + /// + /// Save session file + /// + /// + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern bool llama_save_session_file(SafeLLamaContextHandle ctx, string path_session, llama_token[] tokens, ulong n_token_count); + + /// + /// Run the llama inference to obtain the logits and probabilities for the next token. + /// tokens + n_tokens is the provided batch of new tokens to process + /// n_past is the number of tokens to use from previous eval calls + /// + /// + /// + /// + /// + /// + /// Returns 0 on success + [DllImport(libraryName)] + public static extern int llama_eval(SafeLLamaContextHandle ctx, llama_token[] tokens, int n_tokens, int n_past, int n_threads); + + /// + /// Convert the provided text into tokens. + /// The tokens pointer must be large enough to hold the resulting tokens. + /// Returns the number of tokens on success, no more than n_max_tokens + /// Returns a negative number on failure - the number of tokens that would have been returned + /// + /// + /// + /// + /// + /// + /// + [DllImport(libraryName)] + public static extern int llama_tokenize(SafeLLamaContextHandle ctx, string text, llama_token[] tokens, int n_max_tokens, bool add_bos); + + [DllImport(libraryName)] + public static extern int llama_n_vocab(SafeLLamaContextHandle ctx); + + [DllImport(libraryName)] + public static extern int llama_n_ctx(SafeLLamaContextHandle ctx); + + [DllImport(libraryName)] + public static extern int llama_n_embd(SafeLLamaContextHandle ctx); + + /// + /// Token logits obtained from the last call to llama_eval() + /// The logits for the last token are stored in the last row + /// Can be mutated in order to change the probabilities of the next token + /// Rows: n_tokens + /// Cols: n_vocab + /// + /// + /// + [DllImport(libraryName)] + public static extern float* llama_get_logits(SafeLLamaContextHandle ctx); + + /// + /// Get the embeddings for the input + /// shape: [n_embd] (1-dimensional) + /// + /// + /// + [DllImport(libraryName)] + public static extern float* llama_get_embeddings(SafeLLamaContextHandle ctx); + + /// + /// Token Id -> String. Uses the vocabulary in the provided context + /// + /// + /// + /// Pointer to a string. + [DllImport(libraryName)] + public static extern IntPtr llama_token_to_str(SafeLLamaContextHandle ctx, llama_token token); + + [DllImport(libraryName)] + public static extern llama_token llama_token_bos(); + + [DllImport(libraryName)] + public static extern llama_token llama_token_eos(); + + [DllImport(libraryName)] + public static extern llama_token llama_token_nl(); + + [DllImport(libraryName)] + public static extern void llama_print_timings(SafeLLamaContextHandle ctx); + + [DllImport(libraryName)] + public static extern void llama_reset_timings(SafeLLamaContextHandle ctx); + + /// + /// Print system information + /// + /// + [DllImport(libraryName)] + public static extern IntPtr llama_print_system_info(); + } +} diff --git a/LLama/Native/NativeInfo.cs b/LLama/Native/NativeInfo.cs new file mode 100644 index 00000000..6711db78 --- /dev/null +++ b/LLama/Native/NativeInfo.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace LLama.Native +{ + internal class NativeInfo + { + internal static readonly int LLAMA_FILE_VERSION = 1; + internal static readonly string LLAMA_FILE_MAGIC = "ggjt"; + internal static readonly string LLAMA_FILE_MAGIC_UNVERSIONED = "ggml"; + internal static readonly string LLAMA_SESSION_MAGIC = "ggsn"; + internal static readonly int LLAMA_SESSION_VERSION = 1; + } +} diff --git a/LLama/Native/SafeLLamaContextHandle.cs b/LLama/Native/SafeLLamaContextHandle.cs new file mode 100644 index 00000000..323ecc3d --- /dev/null +++ b/LLama/Native/SafeLLamaContextHandle.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + internal class SafeLLamaContextHandle: SafeLLamaHandleBase + { + protected SafeLLamaContextHandle() + { + } + + public SafeLLamaContextHandle(IntPtr handle) + : base(handle) + { + } + + protected override bool ReleaseHandle() + { + NativeApi.llama_free(handle); + SetHandle(IntPtr.Zero); + return true; + } + } +} diff --git a/LLama/Native/SafeLLamaHandleBase.cs b/LLama/Native/SafeLLamaHandleBase.cs new file mode 100644 index 00000000..a6febf0d --- /dev/null +++ b/LLama/Native/SafeLLamaHandleBase.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + internal abstract class SafeLLamaHandleBase: SafeHandle + { + private protected SafeLLamaHandleBase() + : base(IntPtr.Zero, ownsHandle: true) + { + } + + private protected SafeLLamaHandleBase(IntPtr handle) + : base(IntPtr.Zero, ownsHandle: true) + { + SetHandle(handle); + } + + private protected SafeLLamaHandleBase(IntPtr handle, bool ownsHandle) + : base(IntPtr.Zero, ownsHandle) + { + SetHandle(handle); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + public override string ToString() + => $"0x{handle.ToString("x16")}"; + } +} diff --git a/LLama/Native/SamplingApi.cs b/LLama/Native/SamplingApi.cs new file mode 100644 index 00000000..dc4d45bf --- /dev/null +++ b/LLama/Native/SamplingApi.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text; + +namespace LLama.Native +{ + using llama_token = Int32; + internal unsafe class SamplingApi + { + /// + /// Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + /// + public static void llama_sample_repetition_penalty(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, llama_token[] last_tokens, ulong last_tokens_size, float penalty) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_repetition_penalty(ctx, new IntPtr(&st), last_tokens, last_tokens_size, penalty); + } + + /// + /// Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + /// + /// + public static void llama_sample_frequency_and_presence_penalties(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, llama_token[] last_tokens, ulong last_tokens_size, float alpha_frequency, float alpha_presence) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_frequency_and_presence_penalties(ctx, new IntPtr(&st), last_tokens, last_tokens_size, alpha_frequency, alpha_presence); + } + + /// + /// Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. + /// + /// + /// Pointer to LLamaTokenDataArray + public static void llama_sample_softmax(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_softmax(ctx, new IntPtr(&st)); + } + + /// + /// Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + public static void llama_sample_top_k(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, int k, ulong min_keep) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_top_k(ctx, new IntPtr(&st), k, min_keep); + } + + /// + /// Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + public static void llama_sample_top_p(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float p, ulong min_keep) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_top_p(ctx, new IntPtr(&st), p, min_keep); + } + + /// + /// Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + public static void llama_sample_tail_free(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float z, ulong min_keep) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_tail_free(ctx, new IntPtr(&st), z, min_keep); + } + + /// + /// Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + /// + public static void llama_sample_typical(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float p, ulong min_keep) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_typical(ctx, new IntPtr(&st), p, min_keep); + } + + public static void llama_sample_temperature(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float temp) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + NativeApi.llama_sample_temperature(ctx, new IntPtr(&st), temp); + } + + /// + /// Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// + /// + /// A vector of `LLamaTokenData` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + /// + public static llama_token llama_sample_token_mirostat(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, int m, float[] mu) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + return NativeApi.llama_sample_token_mirostat(ctx, new IntPtr(&st), tau, eta, m, mu); + } + + /// + /// Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// + /// + /// A vector of `LLamaTokenData` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + /// + public static llama_token llama_sample_token_mirostat_v2(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates, float tau, float eta, float[] mu) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + return NativeApi.llama_sample_token_mirostat_v2(ctx, new IntPtr(&st), tau, eta, mu); + } + + /// + /// Selects the token with the highest probability. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + public static llama_token llama_sample_token_greedy(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + return NativeApi.llama_sample_token_greedy(ctx, new IntPtr(&st)); + } + + /// + /// Randomly selects a token from the candidates based on their probabilities. + /// + /// + /// Pointer to LLamaTokenDataArray + /// + public static llama_token llama_sample_token(SafeLLamaContextHandle ctx, LLamaTokenDataArray candidates) + { + var handle = candidates.data.Pin(); + var st = new LLamaTokenDataArrayNative(); + st.data = new IntPtr(handle.Pointer); + st.size = candidates.size; + st.sorted = candidates.sorted; + return NativeApi.llama_sample_token(ctx, new IntPtr(&st)); + } + } +} diff --git a/LLamaSharp.sln b/LLamaSharp.sln new file mode 100644 index 00000000..710532dc --- /dev/null +++ b/LLamaSharp.sln @@ -0,0 +1,51 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.33424.131 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLama.Console", "LLama.Console\LLama.Console.csproj", "{BE303C09-6532-4560-A136-D678C40B0FE6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLamaSharp", "LLama\LLamaSharp.csproj", "{2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLama.Unittest", "LLama.Unittest\LLama.Unittest.csproj", "{BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {BE303C09-6532-4560-A136-D678C40B0FE6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BE303C09-6532-4560-A136-D678C40B0FE6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BE303C09-6532-4560-A136-D678C40B0FE6}.Debug|x64.ActiveCfg = Debug|x64 + {BE303C09-6532-4560-A136-D678C40B0FE6}.Debug|x64.Build.0 = Debug|x64 + {BE303C09-6532-4560-A136-D678C40B0FE6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BE303C09-6532-4560-A136-D678C40B0FE6}.Release|Any CPU.Build.0 = Release|Any CPU + {BE303C09-6532-4560-A136-D678C40B0FE6}.Release|x64.ActiveCfg = Release|x64 + {BE303C09-6532-4560-A136-D678C40B0FE6}.Release|x64.Build.0 = Release|x64 + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Debug|x64.ActiveCfg = Debug|x64 + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Debug|x64.Build.0 = Debug|x64 + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Release|Any CPU.Build.0 = Release|Any CPU + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Release|x64.ActiveCfg = Release|x64 + {2B5A6DEB-FEC2-44C3-AD32-5B4A26A43026}.Release|x64.Build.0 = Release|x64 + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Debug|x64.ActiveCfg = Debug|x64 + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Debug|x64.Build.0 = Debug|x64 + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Release|Any CPU.Build.0 = Release|Any CPU + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Release|x64.ActiveCfg = Release|x64 + {BAC1CFA9-E6AC-4BD0-A548-A8066D3C467E}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {87746B65-249E-4AD9-882B-7B919D079367} + EndGlobalSection +EndGlobal