|
|
|
@@ -15,6 +15,11 @@ public sealed class BatchedExecutor |
|
|
|
private int _nextSequenceId; |
|
|
|
|
|
|
|
internal LLamaBatch Batch { get; } |
|
|
|
|
|
|
|
/// <summary> |
|
|
|
/// Epoch is incremented every time Infer is called. Conversations can use this to keep track of |
|
|
|
/// whether they're waiting for inference, or can be sampled. |
|
|
|
/// </summary> |
|
|
|
internal ulong Epoch { get; private set; } |
|
|
|
|
|
|
|
/// <summary> |
|
|
|
@@ -50,6 +55,11 @@ public sealed class BatchedExecutor |
|
|
|
Epoch = 1; |
|
|
|
} |
|
|
|
|
|
|
|
~BatchedExecutor() |
|
|
|
{ |
|
|
|
Dispose(); |
|
|
|
} |
|
|
|
|
|
|
|
/// <summary> |
|
|
|
/// Start a new <see cref="Conversation"/> with the given prompt |
|
|
|
/// </summary> |
|
|
|
@@ -97,6 +107,8 @@ public sealed class BatchedExecutor |
|
|
|
return; |
|
|
|
IsDisposed = true; |
|
|
|
|
|
|
|
GC.SuppressFinalize(this); |
|
|
|
|
|
|
|
Context.Dispose(); |
|
|
|
} |
|
|
|
|
|
|
|
|