diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj index af47f5c4..3ecacdfe 100644 --- a/LLama.Examples/LLama.Examples.csproj +++ b/LLama.Examples/LLama.Examples.csproj @@ -1,4 +1,4 @@ - + Exe @@ -29,7 +29,7 @@ - + diff --git a/LLama.Unittest/LLama.Unittest.csproj b/LLama.Unittest/LLama.Unittest.csproj index 596e5da9..a62c3201 100644 --- a/LLama.Unittest/LLama.Unittest.csproj +++ b/LLama.Unittest/LLama.Unittest.csproj @@ -1,4 +1,4 @@ - + net6.0 @@ -13,8 +13,8 @@ - - + + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/LLama.WebAPI/LLama.WebAPI.csproj b/LLama.WebAPI/LLama.WebAPI.csproj index 2b14d8a4..bdb2ad8a 100644 --- a/LLama.WebAPI/LLama.WebAPI.csproj +++ b/LLama.WebAPI/LLama.WebAPI.csproj @@ -8,7 +8,7 @@ - + diff --git a/LLama/Native/SafeLLamaGrammarHandle.cs b/LLama/Native/SafeLLamaGrammarHandle.cs index f430b7c3..ee27befd 100644 --- a/LLama/Native/SafeLLamaGrammarHandle.cs +++ b/LLama/Native/SafeLLamaGrammarHandle.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Runtime.CompilerServices; using LLama.Exceptions; using LLama.Grammars; @@ -49,38 +50,39 @@ namespace LLama.Native // Borrow an array large enough to hold every single element // and another array large enough to hold a pointer to each rule var allElements = ArrayPool.Shared.Rent(totalElements); - var pointers = ArrayPool.Shared.Rent(rules.Count); + var rulePointers = ArrayPool.Shared.Rent(rules.Count); try { - fixed (LLamaGrammarElement* allElementsPtr = allElements) + // We're taking pointers into `allElements` below, so this pin is required to fix + // that memory in place while those pointers are in use! + using var pin = allElements.AsMemory().Pin(); + + var elementIndex = 0; + var ruleIndex = 0; + foreach (var rule in rules) { - var elementIndex = 0; - var pointerIndex = 0; - foreach (var rule in rules) - { - // Save a pointer to the start of this rule - pointers[pointerIndex++] = (IntPtr)(allElementsPtr + elementIndex); + // Save a pointer to the start of this rule + rulePointers[ruleIndex++] = (IntPtr)Unsafe.AsPointer(ref allElements[elementIndex]); - // Copy all of the rule elements into the flat array - foreach (var element in rule.Elements) - allElementsPtr[elementIndex++] = element; - } + // Copy all of the rule elements into the flat array + foreach (var element in rule.Elements) + allElements[elementIndex++] = element; + } - // Sanity check some things that should be true if the copy worked as planned - Debug.Assert((ulong)pointerIndex == nrules); - Debug.Assert(elementIndex == totalElements); + // Sanity check some things that should be true if the copy worked as planned + Debug.Assert((ulong)ruleIndex == nrules); + Debug.Assert(elementIndex == totalElements); - // Make the actual call through to llama.cpp - fixed (void* ptr = pointers) - { - return Create((LLamaGrammarElement**)ptr, nrules, start_rule_index); - } + // Make the actual call through to llama.cpp + fixed (void* ptr = rulePointers) + { + return Create((LLamaGrammarElement**)ptr, nrules, start_rule_index); } } finally { ArrayPool.Shared.Return(allElements); - ArrayPool.Shared.Return(pointers); + ArrayPool.Shared.Return(rulePointers); } } }