+ + + + +Use chat session and strip role names +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class ChatSessionStripRoleName +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); + ChatSession session = new ChatSession(ex).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started. The role names won't be printed."); + Console.ForegroundColor = ConsoleColor.White; + + while (true) + { + foreach (var text in session.Chat(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" } })) + { + Console.Write(text); + } + + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } +} + + + + + + + +
+ + + + +Use chat session without removing role names +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class ChatSessionWithRoleName +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); + ChatSession session = new ChatSession(ex); // The only change is to remove the transform for the output text stream. + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result."); + Console.ForegroundColor = ConsoleColor.White; + + // show the prompt + Console.Write(prompt); + while (true) + { + foreach (var text in session.Chat(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" } })) + { + Console.Write(text); + } + + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } +} + + + + + + + +
+ + + + +Get embeddings +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class GetEmbeddings +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var embedder = new LLamaEmbedder(new ModelParams(modelPath)); + + while (true) + { + Console.Write("Please input your text: "); + Console.ForegroundColor = ConsoleColor.Green; + var text = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + + Console.WriteLine(string.Join(", ", embedder.GetEmbeddings(text))); + Console.WriteLine(); + } + } +} + + + + + + + +
+ + + + +Use instruct executor +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class InstructModeExecute +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/dan.txt").Trim(); + + InstructExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024))); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions. For example, you can input \"Write a story about a fox who want to " + + "make friend with human, no less than 200 words.\""); + Console.ForegroundColor = ConsoleColor.White; + + var inferenceParams = new InferenceParams() { Temperature = 0.8f, MaxTokens = 300 }; + + while (true) + { + foreach (var text in ex.Infer(prompt, inferenceParams)) + { + Console.Write(text); + } + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } +} + + + + + + + +
+ + + + +Use interactive executor +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class InteractiveModeExecute +{ + public async static Task Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + + InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 256))); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 64 and the context size is 256. (an example for small scale usage)"); + Console.ForegroundColor = ConsoleColor.White; + + Console.Write(prompt); + + var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" }, MaxTokens = 64 }; + + while (true) + { + await foreach (var text in ex.InferAsync(prompt, inferenceParams)) + { + Console.Write(text); + } + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } +} + + + + + + + +
+ + + + +Load and save chat session +using LLama.Common; +using LLama.OldVersion; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class SaveAndLoadSession +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); + ChatSession session = new ChatSession(ex); // The only change is to remove the transform for the output text stream. + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The chat session has started. In this example, the prompt is printed for better visual result. Input \"save\" to save and reload the session."); + Console.ForegroundColor = ConsoleColor.White; + + // show the prompt + Console.Write(prompt); + while (true) + { + foreach (var text in session.Chat(prompt, new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" } })) + { + Console.Write(text); + } + + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + if (prompt == "save") + { + Console.Write("Preparing to save the state, please input the path you want to save it: "); + Console.ForegroundColor = ConsoleColor.Green; + var statePath = Console.ReadLine(); + session.SaveSession(statePath); + Console.ForegroundColor = ConsoleColor.White; + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Saved session!"); + Console.ForegroundColor = ConsoleColor.White; + + ex.Model.Dispose(); + ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5))); + session = new ChatSession(ex).WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Bob:" }, redundancyLength: 8)); + session.LoadSession(statePath); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Loaded session!"); + Console.ForegroundColor = ConsoleColor.White; + + Console.Write("Now you can continue your session: "); + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } + } +} + + + + + + + +
+ + + + +Load and save model/exeutor state +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class LoadAndSaveState +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + var prompt = File.ReadAllText("Assets/chat-with-bob.txt").Trim(); + + InteractiveExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 256))); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The executor has been enabled. In this example, the prompt is printed, the maximum tokens is set to 64 and the context size is 256. (an example for small scale usage)"); + Console.ForegroundColor = ConsoleColor.White; + + Console.Write(prompt); + + var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "User:" } }; + + while (true) + { + foreach (var text in ex.Infer(prompt, inferenceParams)) + { + Console.Write(text); + } + + prompt = Console.ReadLine(); + if (prompt == "save") + { + Console.Write("Your path to save model state: "); + string modelStatePath = Console.ReadLine(); + ex.Model.SaveState(modelStatePath); + + Console.Write("Your path to save executor state: "); + string executorStatePath = Console.ReadLine(); + ex.SaveState(executorStatePath); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("All states saved!"); + Console.ForegroundColor = ConsoleColor.White; + + var model = ex.Model; + model.LoadState(modelStatePath); + ex = new InteractiveExecutor(model); + ex.LoadState(executorStatePath); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Loaded state!"); + Console.ForegroundColor = ConsoleColor.White; + + Console.Write("Now you can continue your session: "); + Console.ForegroundColor = ConsoleColor.Green; + prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + } + } + } +} + + + + + + + +
+ + + + +Quantize model +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +public class QuantizeModel +{ + public static void Run() + { + Console.Write("Please input your original model path: "); + var inputPath = Console.ReadLine(); + Console.Write("Please input your output model path: "); + var outputPath = Console.ReadLine(); + Console.Write("Please input the quantize type (one of q4_0, q4_1, q5_0, q5_1, q8_0): "); + var quantizeType = Console.ReadLine(); + if (LLamaQuantizer.Quantize(inputPath, outputPath, quantizeType)) + { + Console.WriteLine("Quantization succeed!"); + } + else + { + Console.WriteLine("Quantization failed!"); + } + } +} + + + + + + + +
+ + + + +Use stateless exeutor +using LLama.Common; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +public class StatelessModeExecute +{ + public static void Run() + { + Console.Write("Please input your model path: "); + string modelPath = Console.ReadLine(); + + StatelessExecutor ex = new(new LLamaModel(new ModelParams(modelPath, contextSize: 256))); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The executor has been enabled. In this example, the inference is an one-time job. That says, the previous input and response has " + + "no impact on the current response. Now you can ask it questions. Note that in this example, no prompt was set for LLM and the maximum response tokens is 50. " + + "It may not perform well because of lack of prompt. This is also an example that could indicate the improtance of prompt in LLM. To improve it, you can add " + + "a prompt for it yourself!"); + Console.ForegroundColor = ConsoleColor.White; + + var inferenceParams = new InferenceParams() { Temperature = 0.6f, AntiPrompts = new List<string> { "Question:", "#", "Question: ", ".\n" }, MaxTokens = 50 }; + + while (true) + { + Console.Write("\nQuestion: "); + Console.ForegroundColor = ConsoleColor.Green; + string prompt = Console.ReadLine(); + Console.ForegroundColor = ConsoleColor.White; + Console.Write("Answer: "); + prompt = $"Question: {prompt.Trim()} Answer: "; + foreach (var text in ex.Infer(prompt, inferenceParams)) + { + Console.Write(text); + } + } + } +} + + + + + + + +