From db1bc741b039b6944a61477486a6e3c4ec10d51d Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Tue, 7 Nov 2023 19:41:44 +0000 Subject: [PATCH] Modified `ContextSize` in parameters to be nullable. A null value means autodetect from the model. --- LLama.Web/Common/ModelOptions.cs | 4 +- LLama/Abstractions/IContextParams.cs | 4 +- LLama/Common/ModelParams.cs | 101 ++++++------------- LLama/Extensions/IContextParamsExtensions.cs | 2 +- LLama/Native/LLamaContextParams.cs | 2 +- 5 files changed, 38 insertions(+), 75 deletions(-) diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index 6a63ccc3..8cbf2f09 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -17,9 +17,9 @@ namespace LLama.Web.Common public int MaxInstances { get; set; } /// - /// Model context size (n_ctx) + /// Model context size (n_ctx). Null to use value from model. /// - public uint ContextSize { get; set; } = 512; + public uint? ContextSize { get; set; } /// /// the GPU that is used for scratch and small tensors diff --git a/LLama/Abstractions/IContextParams.cs b/LLama/Abstractions/IContextParams.cs index 8ff6d7cc..a2ac24f1 100644 --- a/LLama/Abstractions/IContextParams.cs +++ b/LLama/Abstractions/IContextParams.cs @@ -8,9 +8,9 @@ namespace LLama.Abstractions; public interface IContextParams { /// - /// Model context size (n_ctx) + /// Model context size (n_ctx). Null to use value from model file. /// - uint ContextSize { get; set; } + uint? ContextSize { get; set; } /// /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index ee5bd3e4..9561e482 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -12,105 +12,68 @@ namespace LLama.Common public record ModelParams : ILLamaParams { - /// - /// Model context size (n_ctx) - /// - public uint ContextSize { get; set; } = 512; - /// - /// the GPU that is used for scratch and small tensors - /// + /// + public uint? ContextSize { get; set; } + + /// public int MainGpu { get; set; } = 0; - /// - /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) - /// + /// public int GpuLayerCount { get; set; } = 20; - /// - /// Seed for the random number generator (seed) - /// + + /// public uint Seed { get; set; } = 0xFFFFFFFF; - /// - /// Use f16 instead of f32 for memory kv (memory_f16) - /// + + /// public bool UseFp16Memory { get; set; } = true; - /// - /// Use mmap for faster loads (use_mmap) - /// + + /// public bool UseMemorymap { get; set; } = true; - /// - /// Use mlock to keep model in memory (use_mlock) - /// + + /// public bool UseMemoryLock { get; set; } - /// - /// Compute perplexity over the prompt (perplexity) - /// + + /// public bool Perplexity { get; set; } - /// - /// Model path (model) - /// + + /// public string ModelPath { get; set; } - /// - /// List of LoRAs to apply - /// + /// public AdapterCollection LoraAdapters { get; set; } = new(); - /// - /// base model path for the lora adapter (lora_base) - /// + /// public string LoraBase { get; set; } = string.Empty; - /// - /// Number of threads (null = autodetect) (n_threads) - /// + /// public uint? Threads { get; set; } - /// - /// Number of threads to use for batch processing (null = autodetect) (n_threads) - /// + /// public uint? BatchThreads { get; set; } - /// - /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch) - /// + /// public uint BatchSize { get; set; } = 512; - /// - /// Whether to use embedding mode. (embedding) Note that if this is set to true, - /// The LLamaModel won't produce text response anymore. - /// + /// public bool EmbeddingMode { get; set; } - /// - /// how split tensors should be distributed across GPUs. - /// - /// "[ 3, 2 ]" will assign 60% of the data to GPU 0 and 40% to GPU 1. + /// [JsonConverter(typeof(TensorSplitsCollectionConverter))] public TensorSplitsCollection TensorSplits { get; set; } = new(); - /// - /// RoPE base frequency - /// - public float? RopeFrequencyBase { get; set; } + /// + public float? RopeFrequencyBase { get; set; } - /// - /// RoPE frequency scaling factor - /// - public float? RopeFrequencyScale { get; set; } + /// + public float? RopeFrequencyScale { get; set; } - /// - /// Use experimental mul_mat_q kernels - /// - public bool MulMatQ { get; set; } + /// + public bool MulMatQ { get; set; } - /// - /// Load vocab only (no weights) - /// + /// public bool VocabOnly { get; set; } - /// - /// The encoding to use to convert text for the model - /// + /// [JsonConverter(typeof(EncodingConverter))] public Encoding Encoding { get; set; } = Encoding.UTF8; diff --git a/LLama/Extensions/IContextParamsExtensions.cs b/LLama/Extensions/IContextParamsExtensions.cs index fcc9d372..ed59c9df 100644 --- a/LLama/Extensions/IContextParamsExtensions.cs +++ b/LLama/Extensions/IContextParamsExtensions.cs @@ -21,7 +21,7 @@ namespace LLama.Extensions public static void ToLlamaContextParams(this IContextParams @params, out LLamaContextParams result) { result = NativeApi.llama_context_default_params(); - result.n_ctx = @params.ContextSize; + result.n_ctx = @params.ContextSize ?? 0; result.n_batch = @params.BatchSize; result.seed = @params.Seed; result.f16_kv = @params.UseFp16Memory; diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs index 0a397a3d..9a0b2a8e 100644 --- a/LLama/Native/LLamaContextParams.cs +++ b/LLama/Native/LLamaContextParams.cs @@ -22,7 +22,7 @@ namespace LLama.Native public uint seed; /// - /// text context + /// text context, 0 = from model /// public uint n_ctx;