diff --git a/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj b/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj new file mode 100644 index 00000000..1575e3eb --- /dev/null +++ b/LLama.KernelMemory/LLamaSharp.KernelMemory.csproj @@ -0,0 +1,17 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + diff --git a/LLama.KernelMemory/LLamaSharpTextEmbeddingGeneration.cs b/LLama.KernelMemory/LLamaSharpTextEmbeddingGeneration.cs new file mode 100644 index 00000000..7ddd42c5 --- /dev/null +++ b/LLama.KernelMemory/LLamaSharpTextEmbeddingGeneration.cs @@ -0,0 +1,45 @@ +using LLama; +using LLama.Common; +using Microsoft.SemanticKernel.AI.Embeddings; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLamaSharp.KernelMemory +{ + public class LLamaSharpTextEmbeddingGeneration : ITextEmbeddingGeneration, IDisposable + { + private readonly LlamaSharpConfig _config; + private readonly LLamaEmbedder _embedder; + private readonly LLamaWeights _weights; + + public LLamaSharpTextEmbeddingGeneration(LlamaSharpConfig config) + { + this._config = config; + var @params = new ModelParams(_config.ModelPath); + _weights = LLamaWeights.LoadFromFile(@params); + _embedder = new LLamaEmbedder(_weights, @params); + } + + public void Dispose() + { + _embedder.Dispose(); + _weights.Dispose(); + } + + public Task>> GenerateEmbeddingsAsync(IList data, CancellationToken cancellationToken = default) + { + IList> results = new List>(); + + foreach (var d in data) + { + var embeddings = _embedder.GetEmbeddings(d); + results.Add(new ReadOnlyMemory(embeddings)); + } + + return Task.FromResult(results); + } + } +} diff --git a/LLama.KernelMemory/LlamaSharpConfig.cs b/LLama.KernelMemory/LlamaSharpConfig.cs new file mode 100644 index 00000000..b3cc5782 --- /dev/null +++ b/LLama.KernelMemory/LlamaSharpConfig.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLamaSharp.KernelMemory +{ + public class LlamaSharpConfig + { + public LlamaSharpConfig(string modelPath) + { + ModelPath = modelPath; + } + + public string ModelPath { get; set; } + public uint? ContextSize { get; set; } + public uint? Seed { get; set; } + public int? GpuLayerCount { get; set; } + } +} diff --git a/LLama.KernelMemory/LlamaSharpTextGeneration.cs b/LLama.KernelMemory/LlamaSharpTextGeneration.cs new file mode 100644 index 00000000..90809a8a --- /dev/null +++ b/LLama.KernelMemory/LlamaSharpTextGeneration.cs @@ -0,0 +1,58 @@ +using LLama; +using LLama.Common; +using Microsoft.SemanticMemory.AI; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace LLamaSharp.KernelMemory +{ + public class LlamaSharpTextGeneration : ITextGeneration, IDisposable + { + private readonly LlamaSharpConfig _config; + private readonly LLamaWeights _weights; + private readonly InstructExecutor _executor; + private readonly LLamaContext _context; + + public LlamaSharpTextGeneration(LlamaSharpConfig config) + { + this._config = config; + var parameters = new ModelParams(config.ModelPath) + { + ContextSize = config?.ContextSize ?? 1024, + Seed = config?.Seed ?? 0, + GpuLayerCount = config?.GpuLayerCount ?? 20 + }; + _weights = LLamaWeights.LoadFromFile(parameters); + _context = _weights.CreateContext(parameters); + _executor = new InstructExecutor(_context); + + } + + public void Dispose() + { + _context.Dispose(); + _weights.Dispose(); + } + + public IAsyncEnumerable GenerateTextAsync(string prompt, TextGenerationOptions options, CancellationToken cancellationToken = default) + { + return _executor.InferAsync(prompt, OptionsToParams(options), cancellationToken: cancellationToken); + } + + private static InferenceParams OptionsToParams(TextGenerationOptions options) + { + return new InferenceParams() + { + AntiPrompts = options.StopSequences, + Temperature = (float)options.Temperature, + MaxTokens = options.MaxTokens ?? 1024, + FrequencyPenalty = (float)options.FrequencyPenalty, + PresencePenalty = (float)options.PresencePenalty, + TopP = (float)options.TopP, + }; + } + } +} diff --git a/LLamaSharp.sln b/LLamaSharp.sln index 2a039d41..76334657 100644 --- a/LLamaSharp.sln +++ b/LLamaSharp.sln @@ -15,6 +15,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLama.Web", "LLama.Web\LLam EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLamaSharp.SemanticKernel", "LLama.SemanticKernel\LLamaSharp.SemanticKernel.csproj", "{D98F93E3-B344-4F9D-86BB-FDBF6768B587}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LLamaSharp.KernelMemory", "LLama.KernelMemory\LLamaSharp.KernelMemory.csproj", "{E5589AE7-B86F-4343-A1CC-8E5D34596E52}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -97,6 +99,18 @@ Global {D98F93E3-B344-4F9D-86BB-FDBF6768B587}.Release|Any CPU.Build.0 = Release|Any CPU {D98F93E3-B344-4F9D-86BB-FDBF6768B587}.Release|x64.ActiveCfg = Release|Any CPU {D98F93E3-B344-4F9D-86BB-FDBF6768B587}.Release|x64.Build.0 = Release|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Debug|x64.ActiveCfg = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Debug|x64.Build.0 = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.GPU|Any CPU.ActiveCfg = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.GPU|Any CPU.Build.0 = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.GPU|x64.ActiveCfg = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.GPU|x64.Build.0 = Debug|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Release|Any CPU.Build.0 = Release|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Release|x64.ActiveCfg = Release|Any CPU + {E5589AE7-B86F-4343-A1CC-8E5D34596E52}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE