From af680ac2d71cc552c3cde1ab09e98f6622490aa7 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Sat, 2 Sep 2023 02:04:11 +0100 Subject: [PATCH] Created a hierarchy of exceptions for grammar format issues. This allows the base catch-all exception to be caught for general handling, or more specific exceptions to be caught for more specific handling. --- LLama/Exceptions/GrammarFormatException.cs | 21 ---- LLama/Exceptions/GrammarFormatExceptions.cs | 125 ++++++++++++++++++++ LLama/Grammars/GBNFGrammarParser.cs | 23 ++-- LLama/Grammars/Grammar.cs | 23 ++-- LLama/Grammars/GrammarRule.cs | 13 +- 5 files changed, 150 insertions(+), 55 deletions(-) delete mode 100644 LLama/Exceptions/GrammarFormatException.cs create mode 100644 LLama/Exceptions/GrammarFormatExceptions.cs diff --git a/LLama/Exceptions/GrammarFormatException.cs b/LLama/Exceptions/GrammarFormatException.cs deleted file mode 100644 index 5b1299dd..00000000 --- a/LLama/Exceptions/GrammarFormatException.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace LLama.Exceptions -{ - public class GrammarFormatException - : Exception - { - public GrammarFormatException() - { - - } - - public GrammarFormatException(string message) - : base(message) - { - - } - } -} diff --git a/LLama/Exceptions/GrammarFormatExceptions.cs b/LLama/Exceptions/GrammarFormatExceptions.cs new file mode 100644 index 00000000..62b75224 --- /dev/null +++ b/LLama/Exceptions/GrammarFormatExceptions.cs @@ -0,0 +1,125 @@ +using System; + +namespace LLama.Exceptions; + +/// +/// Base class for all grammar exceptions +/// +public abstract class GrammarFormatException + : Exception +{ + internal GrammarFormatException(string message) + : base(message) + { + } +} + + +/// +/// An incorrect number of characters were encountered while parsing a hex literal +/// +public class GrammarUnexpectedHexCharsCount + : GrammarFormatException +{ + internal GrammarUnexpectedHexCharsCount(int size, string source) + : base($"Expecting {size} hex chars at {source}") + { + } +} + +/// +/// Failed to parse a "name" element when one was expected +/// +public class GrammarExpectedName + : GrammarFormatException +{ + internal GrammarExpectedName(string source) + : base($"Expecting name at {source}") + { + } +} + +/// +/// An unexpected character was encountered after an escape sequence +/// +public class GrammarUnknownEscapeCharacter + : GrammarFormatException +{ + internal GrammarUnknownEscapeCharacter(string source) + : base($"Unknown escape at {source}") + { + } +} + +/// +/// End-of-file was encountered while parsing +/// +public class GrammarUnexpectedEndOfInput + : GrammarFormatException +{ + internal GrammarUnexpectedEndOfInput() + : base($"Unexpected end of input") + { + } +} + +/// +/// A specified string was expected when parsing +/// +public class GrammarExpectedNext + : GrammarFormatException +{ + internal GrammarExpectedNext(string expected, string source) + : base($"Expected '{expected}' at {source}") + { + } +} + +/// +/// A specified character was expected to preceded another when parsing +/// +public class GrammarExpectedPrevious + : GrammarFormatException +{ + internal GrammarExpectedPrevious(string expected, string source) + : base($"Expecting preceding item to be '{expected}' at {source}") + { + } +} + + +/// +/// A CHAR_ALT was created without a preceding CHAR element +/// +public class GrammarUnexpectedCharAltElement + : GrammarFormatException +{ + internal GrammarUnexpectedCharAltElement(string ruleId, int index) + : base($"LLamaGrammarElementType.CHAR_ALT without preceding char: {ruleId},{index}") + { + } +} + +/// +/// A CHAR_RNG was created without a preceding CHAR element +/// +public class GrammarUnexpectedCharRngElement + : GrammarFormatException +{ + internal GrammarUnexpectedCharRngElement(string ruleId, int index) + : base($"LLamaGrammarElementType.CHAR_RNG_UPPER without preceding char: {ruleId},{index}") + { + } +} + +/// +/// An END was encountered before the last element +/// +public class GrammarUnexpectedEndElement + : GrammarFormatException +{ + internal GrammarUnexpectedEndElement(string ruleId, int index) + : base($"Unexpected LLamaGrammarElementType.END: {ruleId},{index}") + { + } +} \ No newline at end of file diff --git a/LLama/Grammars/GBNFGrammarParser.cs b/LLama/Grammars/GBNFGrammarParser.cs index aec58c7a..cd5969e4 100644 --- a/LLama/Grammars/GBNFGrammarParser.cs +++ b/LLama/Grammars/GBNFGrammarParser.cs @@ -109,7 +109,7 @@ namespace LLama.Grammars if (pos != end) { - throw new GrammarFormatException($"Expecting {size} hex chars at {Encoding.UTF8.GetString(src.ToArray())}"); + throw new GrammarUnexpectedHexCharsCount(size, Encoding.UTF8.GetString(src.ToArray())); } src = src.Slice(pos); return value; @@ -146,7 +146,7 @@ namespace LLama.Grammars } if (pos == 0) { - throw new GrammarFormatException($"Expecting name at {Encoding.UTF8.GetString(src.ToArray())}"); + throw new GrammarExpectedName(Encoding.UTF8.GetString(src.ToArray())); } return src.Slice(pos); } @@ -177,7 +177,7 @@ namespace LLama.Grammars case (byte)']': return chr; default: - throw new GrammarFormatException("Unknown escape at " + Encoding.UTF8.GetString(src.ToArray())); + throw new GrammarUnknownEscapeCharacter(Encoding.UTF8.GetString(src.ToArray())); } } else if (!src.IsEmpty) @@ -185,7 +185,7 @@ namespace LLama.Grammars return DecodeUTF8(ref src); } - throw new GrammarFormatException("Unexpected end of input"); + throw new GrammarUnexpectedEndOfInput(); } private ReadOnlySpan ParseSequence( @@ -258,17 +258,13 @@ namespace LLama.Grammars // output reference to synthesized rule outElements.Add(new LLamaGrammarElement(LLamaGrammarElementType.RULE_REF, subRuleId)); if (pos[0] != ')') - { - throw new GrammarFormatException($"Expecting ')' at {Encoding.UTF8.GetString(pos.ToArray())}"); - } + throw new GrammarExpectedNext(")", Encoding.UTF8.GetString(pos.ToArray())); pos = ParseSpace(pos.Slice(1), isNested); } else if (pos[0] == '*' || pos[0] == '+' || pos[0] == '?') // repetition operator { if (lastSymStart == outElements.Count) - { - throw new GrammarFormatException($"Expecting preceding item to */+/? at {Encoding.UTF8.GetString(pos.ToArray())}"); - } + throw new GrammarExpectedPrevious("*/+/?", Encoding.UTF8.GetString(pos.ToArray())); // apply transformation to previous symbol (lastSymStart to end) according to // rewrite rules: @@ -349,9 +345,8 @@ namespace LLama.Grammars string name = Encoding.UTF8.GetString(src.Slice(0, nameLen).ToArray()); if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) - { - throw new GrammarFormatException($"Expecting ::= at {Encoding.UTF8.GetString(pos.ToArray())}"); - } + throw new GrammarExpectedNext("::=", Encoding.UTF8.GetString(pos.ToArray())); + pos = ParseSpace(pos.Slice(3), true); pos = ParseAlternates(state, pos, name, ruleId, false); @@ -366,7 +361,7 @@ namespace LLama.Grammars } else if (!pos.IsEmpty) { - throw new GrammarFormatException($"Expecting newline or end at {Encoding.UTF8.GetString(pos.ToArray())}"); + throw new GrammarExpectedNext("newline or EOF", Encoding.UTF8.GetString(pos.ToArray())); } return ParseSpace(pos, true); } diff --git a/LLama/Grammars/Grammar.cs b/LLama/Grammars/Grammar.cs index f8bd9052..dbb3658e 100644 --- a/LLama/Grammars/Grammar.cs +++ b/LLama/Grammars/Grammar.cs @@ -70,10 +70,10 @@ namespace LLama.Grammars private void PrintGrammar(StringBuilder output) { for (var i = 0; i < Rules.Count; i++) - PrintRule(output, (uint)i, Rules[i]); + PrintRule(output, Rules[i]); } - private void PrintRule(StringBuilder output, uint ruleId, GrammarRule rule) + private void PrintRule(StringBuilder output, GrammarRule rule) { output.Append($"{rule.Name} ::= "); @@ -82,37 +82,34 @@ namespace LLama.Grammars var elem = rule.Elements[i]; switch (elem.Type) { + // GrammarRule has already verified that END is not being misused, no need to check again case LLamaGrammarElementType.END: - throw new GrammarFormatException($"Unexpected end of rule: {ruleId}, {i}"); + break; + case LLamaGrammarElementType.ALT: output.Append("| "); break; + case LLamaGrammarElementType.RULE_REF: output.Append($"{Rules[(int)elem.Value].Name} "); break; + case LLamaGrammarElementType.CHAR: output.Append('['); PrintGrammarChar(output, elem.Value); break; + case LLamaGrammarElementType.CHAR_NOT: output.Append("[^"); PrintGrammarChar(output, elem.Value); break; + case LLamaGrammarElementType.CHAR_RNG_UPPER: - if (i == 0 || !rule.Elements[i - 1].IsCharElement()) - { - throw new GrammarFormatException( - $"LLamaGrammarElementType.CHAR_RNG_UPPER without preceding char: {ruleId},{i}"); - } output.Append('-'); PrintGrammarChar(output, elem.Value); break; + case LLamaGrammarElementType.CHAR_ALT: - if (i == 0 || !rule.Elements[i - 1].IsCharElement()) - { - throw new GrammarFormatException( - $"LLamaGrammarElementType.CHAR_ALT without preceding char: {ruleId},{i}"); - } PrintGrammarChar(output, elem.Value); break; diff --git a/LLama/Grammars/GrammarRule.cs b/LLama/Grammars/GrammarRule.cs index beab1078..c57b7084 100644 --- a/LLama/Grammars/GrammarRule.cs +++ b/LLama/Grammars/GrammarRule.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using LLama.Exceptions; using LLama.Native; namespace LLama.Grammars @@ -36,7 +37,7 @@ namespace LLama.Grammars private static void Validate(IReadOnlyList elements, string name) { if (elements.Count == 0) - throw new ArgumentException("Cannot create a GrammaRule with zero elements", nameof(elements)); + throw new ArgumentException("Cannot create a GrammarRule with zero elements", nameof(elements)); if (elements[elements.Count - 1].Type != LLamaGrammarElementType.END) throw new ArgumentException("Last grammar element must be END", nameof(elements)); @@ -46,18 +47,16 @@ namespace LLama.Grammars { case LLamaGrammarElementType.END: if (i != elements.Count - 1) - throw new ArgumentException("Found more than one END grammar element", nameof(elements)); + throw new GrammarUnexpectedEndElement(name, i); continue; case LLamaGrammarElementType.CHAR_RNG_UPPER: if (i == 0 || !elements[i - 1].IsCharElement()) - throw new ArgumentException($"LLamaGrammarElementType.CHAR_RNG_UPPER without preceding char: {name},{i}", nameof(elements)); + throw new GrammarUnexpectedCharRngElement(name, i); break; case LLamaGrammarElementType.CHAR_ALT: if (i == 0 || !elements[i - 1].IsCharElement()) - { - throw new ArgumentException($"LLamaGrammarElementType.CHAR_ALT without preceding char: {name},{i}", nameof(elements)); - } + throw new GrammarUnexpectedCharAltElement(name, i); break; case LLamaGrammarElementType.ALT: @@ -67,7 +66,7 @@ namespace LLama.Grammars break; default: - throw new ArgumentException($"Unknown grammar element type: '{elements[i].Type}'"); + throw new ArgumentException($"Unknown grammar element type: '{elements[i].Type}'", nameof(elements)); } } }