diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs
index 6a63ccc3..8cbf2f09 100644
--- a/LLama.Web/Common/ModelOptions.cs
+++ b/LLama.Web/Common/ModelOptions.cs
@@ -17,9 +17,9 @@ namespace LLama.Web.Common
public int MaxInstances { get; set; }
///
- /// Model context size (n_ctx)
+ /// Model context size (n_ctx). Null to use value from model.
///
- public uint ContextSize { get; set; } = 512;
+ public uint? ContextSize { get; set; }
///
/// the GPU that is used for scratch and small tensors
diff --git a/LLama/Abstractions/IContextParams.cs b/LLama/Abstractions/IContextParams.cs
index 8ff6d7cc..a2ac24f1 100644
--- a/LLama/Abstractions/IContextParams.cs
+++ b/LLama/Abstractions/IContextParams.cs
@@ -8,9 +8,9 @@ namespace LLama.Abstractions;
public interface IContextParams
{
///
- /// Model context size (n_ctx)
+ /// Model context size (n_ctx). Null to use value from model file.
///
- uint ContextSize { get; set; }
+ uint? ContextSize { get; set; }
///
/// batch size for prompt processing (must be >=32 to use BLAS) (n_batch)
diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs
index ee5bd3e4..9561e482 100644
--- a/LLama/Common/ModelParams.cs
+++ b/LLama/Common/ModelParams.cs
@@ -12,105 +12,68 @@ namespace LLama.Common
public record ModelParams
: ILLamaParams
{
- ///
- /// Model context size (n_ctx)
- ///
- public uint ContextSize { get; set; } = 512;
- ///
- /// the GPU that is used for scratch and small tensors
- ///
+ ///
+ public uint? ContextSize { get; set; }
+
+ ///
public int MainGpu { get; set; } = 0;
- ///
- /// Number of layers to run in VRAM / GPU memory (n_gpu_layers)
- ///
+ ///
public int GpuLayerCount { get; set; } = 20;
- ///
- /// Seed for the random number generator (seed)
- ///
+
+ ///
public uint Seed { get; set; } = 0xFFFFFFFF;
- ///
- /// Use f16 instead of f32 for memory kv (memory_f16)
- ///
+
+ ///
public bool UseFp16Memory { get; set; } = true;
- ///
- /// Use mmap for faster loads (use_mmap)
- ///
+
+ ///
public bool UseMemorymap { get; set; } = true;
- ///
- /// Use mlock to keep model in memory (use_mlock)
- ///
+
+ ///
public bool UseMemoryLock { get; set; }
- ///
- /// Compute perplexity over the prompt (perplexity)
- ///
+
+ ///
public bool Perplexity { get; set; }
- ///
- /// Model path (model)
- ///
+
+ ///
public string ModelPath { get; set; }
- ///
- /// List of LoRAs to apply
- ///
+ ///
public AdapterCollection LoraAdapters { get; set; } = new();
- ///
- /// base model path for the lora adapter (lora_base)
- ///
+ ///
public string LoraBase { get; set; } = string.Empty;
- ///
- /// Number of threads (null = autodetect) (n_threads)
- ///
+ ///
public uint? Threads { get; set; }
- ///
- /// Number of threads to use for batch processing (null = autodetect) (n_threads)
- ///
+ ///
public uint? BatchThreads { get; set; }
- ///
- /// batch size for prompt processing (must be >=32 to use BLAS) (n_batch)
- ///
+ ///
public uint BatchSize { get; set; } = 512;
- ///
- /// Whether to use embedding mode. (embedding) Note that if this is set to true,
- /// The LLamaModel won't produce text response anymore.
- ///
+ ///
public bool EmbeddingMode { get; set; }
- ///
- /// how split tensors should be distributed across GPUs.
- ///
- /// "[ 3, 2 ]" will assign 60% of the data to GPU 0 and 40% to GPU 1.
+ ///
[JsonConverter(typeof(TensorSplitsCollectionConverter))]
public TensorSplitsCollection TensorSplits { get; set; } = new();
- ///
- /// RoPE base frequency
- ///
- public float? RopeFrequencyBase { get; set; }
+ ///
+ public float? RopeFrequencyBase { get; set; }
- ///
- /// RoPE frequency scaling factor
- ///
- public float? RopeFrequencyScale { get; set; }
+ ///
+ public float? RopeFrequencyScale { get; set; }
- ///
- /// Use experimental mul_mat_q kernels
- ///
- public bool MulMatQ { get; set; }
+ ///
+ public bool MulMatQ { get; set; }
- ///
- /// Load vocab only (no weights)
- ///
+ ///
public bool VocabOnly { get; set; }
- ///
- /// The encoding to use to convert text for the model
- ///
+ ///
[JsonConverter(typeof(EncodingConverter))]
public Encoding Encoding { get; set; } = Encoding.UTF8;
diff --git a/LLama/Extensions/IContextParamsExtensions.cs b/LLama/Extensions/IContextParamsExtensions.cs
index fcc9d372..ed59c9df 100644
--- a/LLama/Extensions/IContextParamsExtensions.cs
+++ b/LLama/Extensions/IContextParamsExtensions.cs
@@ -21,7 +21,7 @@ namespace LLama.Extensions
public static void ToLlamaContextParams(this IContextParams @params, out LLamaContextParams result)
{
result = NativeApi.llama_context_default_params();
- result.n_ctx = @params.ContextSize;
+ result.n_ctx = @params.ContextSize ?? 0;
result.n_batch = @params.BatchSize;
result.seed = @params.Seed;
result.f16_kv = @params.UseFp16Memory;
diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs
index 0a397a3d..9a0b2a8e 100644
--- a/LLama/Native/LLamaContextParams.cs
+++ b/LLama/Native/LLamaContextParams.cs
@@ -22,7 +22,7 @@ namespace LLama.Native
public uint seed;
///
- /// text context
+ /// text context, 0 = from model
///
public uint n_ctx;