From 703df2e3f3901ee3e508dec40f80fd33a7dda26c Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Fri, 10 May 2024 15:12:31 +0100 Subject: [PATCH] Renamed one of the commented out pre tokenization items, showing the pattern they should all use. --- LLama/Native/llama_vocab_pre_type.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/LLama/Native/llama_vocab_pre_type.cs b/LLama/Native/llama_vocab_pre_type.cs index 5d075d67..7f08fb35 100644 --- a/LLama/Native/llama_vocab_pre_type.cs +++ b/LLama/Native/llama_vocab_pre_type.cs @@ -6,7 +6,12 @@ ///// llama_vocab_pre_type //public enum llama_vocab_pre_type //{ -// LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +// /// +// /// Default pre tokenization type +// /// +// /// LLAMA_VOCAB_PRE_TYPE_DEFAULT +// Default = 0, +// // LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, // LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, // LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,