From 2386be58c4f9f58b7ab78fddac04b492fce38898 Mon Sep 17 00:00:00 2001 From: Marlon Regenhardt Date: Sun, 27 Aug 2023 01:31:27 +0200 Subject: [PATCH 1/3] Example: Add code assistant using Code Llama-7b-Instruct as default --- LLama.Examples/NewVersion/CodingAssistant.cs | 95 ++++++++++++++++++++ LLama.Examples/NewVersion/TestRunner.cs | 5 ++ 2 files changed, 100 insertions(+) create mode 100644 LLama.Examples/NewVersion/CodingAssistant.cs diff --git a/LLama.Examples/NewVersion/CodingAssistant.cs b/LLama.Examples/NewVersion/CodingAssistant.cs new file mode 100644 index 00000000..c599702e --- /dev/null +++ b/LLama.Examples/NewVersion/CodingAssistant.cs @@ -0,0 +1,95 @@ +namespace LLama.Examples.NewVersion +{ + using LLama.Common; + using System; + using System.Reflection; + + internal class CodingAssistant + { + const string DefaultModelUri = "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGML/resolve/main/codellama-7b-instruct.ggmlv3.Q4_K_S.bin"; + + // Source paper with example prompts: + // https://scontent-ham3-1.xx.fbcdn.net/v/t39.2365-6/369856151_1754812304950972_1159666448927483931_n.pdf?_nc_cat=107&ccb=1-7&_nc_sid=3c67a6&_nc_ohc=wURKmnWKaloAX9CL8rD&_nc_ht=scontent-ham3-1.xx&oh=00_AfBSvnWP6BkLgXzZ0OvLGkiDbkejxoM03Xg2ghVhn_InZQ&oe=64EEAC4F + const string InstructionPrefix = "[INST]"; + const string InstructionSuffix = "[/INST]"; + const string SystemInstruction = "You're an intelligent, concise coding assistant. Wrap code in ``` for readability. Don't repeat yourself. Use best practice and good coding standards."; + private static string ModelsDirectory = Path.Combine(Directory.GetParent(Assembly.GetExecutingAssembly().Location)!.FullName, "Models"); + + public static async Task Run() + { + Console.Write("Please input your model path (if left empty, a default model will be downloaded for you): "); + var modelPath = Console.ReadLine(); + + if(string.IsNullOrWhiteSpace(modelPath) ) + { + modelPath = await GetDefaultModel(); + } + + var parameters = new ModelParams(modelPath) + { + ContextSize = 4096, + Seed = 1337, + GpuLayerCount = 5 + }; + using var model = LLamaWeights.LoadFromFile(parameters); + using var context = model.CreateContext(parameters); + var executor = new InstructExecutor(context, InstructionPrefix, InstructionSuffix); + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions." + + "It's a 7B Code Llama, so it's trained for programming tasks like \"Write a C# function reading a file name from a given URI\" or \"Write some programming interview questions\"."); + Console.ForegroundColor = ConsoleColor.White; + + var inferenceParams = new InferenceParams() { + Temperature = 0.8f, + MaxTokens = -1, + }; + + string instruction = $"{SystemInstruction}\n"; + await Console.Out.WriteAsync("Instruction: "); + instruction += Console.ReadLine() ?? "Ask me for instructions."; + while (true) + { + + Console.ForegroundColor = ConsoleColor.Green; + foreach (var text in executor.Infer(instruction+System.Environment.NewLine, inferenceParams)) + { + Console.Write(text); + } + Console.ForegroundColor = ConsoleColor.White; + + await Console.Out.WriteAsync("Instruction: "); + instruction = Console.ReadLine() ?? "Ask me for instructions."; + } + } + + private static async Task GetDefaultModel() + { + var uri = new Uri(DefaultModelUri); + var modelName = uri.Segments[^1]; + await Console.Out.WriteLineAsync($"The following model will be used: {modelName}"); + var modelPath = Path.Combine(ModelsDirectory, modelName); + if(!Directory.Exists(ModelsDirectory)) + { + Directory.CreateDirectory(ModelsDirectory); + } + + if (File.Exists(modelPath)) + { + await Console.Out.WriteLineAsync($"Existing model found, using {modelPath}"); + } + else + { + await Console.Out.WriteLineAsync($"Model not found locally, downloading {DefaultModelUri}..."); + using var http = new HttpClient(); + await using var downloadStream = await http.GetStreamAsync(uri); + await using var fileStream = new FileStream(modelPath, FileMode.Create, FileAccess.Write); + await downloadStream.CopyToAsync(fileStream); + await Console.Out.WriteLineAsync($"Model downloaded and saved to {modelPath}"); + } + + + return modelPath; + } + } +} diff --git a/LLama.Examples/NewVersion/TestRunner.cs b/LLama.Examples/NewVersion/TestRunner.cs index 07f61422..1f77524a 100644 --- a/LLama.Examples/NewVersion/TestRunner.cs +++ b/LLama.Examples/NewVersion/TestRunner.cs @@ -21,6 +21,7 @@ Console.WriteLine("11: Semantic Kernel Prompt."); Console.WriteLine("12: Semantic Kernel Chat."); Console.WriteLine("13: Semantic Kernel Memory."); + Console.WriteLine("14: Coding Assistant."); while (true) { @@ -83,6 +84,10 @@ { await SemanticKernelMemory.Run(); } + else if(choice == 14) + { + await CodingAssistant.Run(); + } else { Console.WriteLine("Cannot parse your choice. Please select again."); From 71f4e2ae4ab24bfef76721148fc4e8dbc97eb700 Mon Sep 17 00:00:00 2001 From: Regenhardt Marlon Date: Wed, 13 Sep 2023 15:45:44 +0200 Subject: [PATCH 2/3] Coding Assistent: Use GGUF model --- LLama.Examples/NewVersion/CodingAssistant.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LLama.Examples/NewVersion/CodingAssistant.cs b/LLama.Examples/NewVersion/CodingAssistant.cs index c599702e..246249c5 100644 --- a/LLama.Examples/NewVersion/CodingAssistant.cs +++ b/LLama.Examples/NewVersion/CodingAssistant.cs @@ -6,7 +6,7 @@ internal class CodingAssistant { - const string DefaultModelUri = "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGML/resolve/main/codellama-7b-instruct.ggmlv3.Q4_K_S.bin"; + const string DefaultModelUri = "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_S.gguf"; // Source paper with example prompts: // https://scontent-ham3-1.xx.fbcdn.net/v/t39.2365-6/369856151_1754812304950972_1159666448927483931_n.pdf?_nc_cat=107&ccb=1-7&_nc_sid=3c67a6&_nc_ohc=wURKmnWKaloAX9CL8rD&_nc_ht=scontent-ham3-1.xx&oh=00_AfBSvnWP6BkLgXzZ0OvLGkiDbkejxoM03Xg2ghVhn_InZQ&oe=64EEAC4F @@ -27,9 +27,7 @@ var parameters = new ModelParams(modelPath) { - ContextSize = 4096, - Seed = 1337, - GpuLayerCount = 5 + ContextSize = 4096 }; using var model = LLamaWeights.LoadFromFile(parameters); using var context = model.CreateContext(parameters); From 0049dcfb66b02dac1b73f83d8ccb992cac18df2a Mon Sep 17 00:00:00 2001 From: Regenhardt Marlon Date: Wed, 13 Sep 2023 17:04:08 +0200 Subject: [PATCH 3/3] Coding Assistent: Improve user information and link to source paper --- LLama.Examples/NewVersion/CodingAssistant.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/LLama.Examples/NewVersion/CodingAssistant.cs b/LLama.Examples/NewVersion/CodingAssistant.cs index 246249c5..9e7dc9d7 100644 --- a/LLama.Examples/NewVersion/CodingAssistant.cs +++ b/LLama.Examples/NewVersion/CodingAssistant.cs @@ -9,7 +9,7 @@ const string DefaultModelUri = "https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_S.gguf"; // Source paper with example prompts: - // https://scontent-ham3-1.xx.fbcdn.net/v/t39.2365-6/369856151_1754812304950972_1159666448927483931_n.pdf?_nc_cat=107&ccb=1-7&_nc_sid=3c67a6&_nc_ohc=wURKmnWKaloAX9CL8rD&_nc_ht=scontent-ham3-1.xx&oh=00_AfBSvnWP6BkLgXzZ0OvLGkiDbkejxoM03Xg2ghVhn_InZQ&oe=64EEAC4F + // https://doi.org/10.48550/arXiv.2308.12950 const string InstructionPrefix = "[INST]"; const string InstructionSuffix = "[/INST]"; const string SystemInstruction = "You're an intelligent, concise coding assistant. Wrap code in ``` for readability. Don't repeat yourself. Use best practice and good coding standards."; @@ -35,7 +35,8 @@ Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions." + - "It's a 7B Code Llama, so it's trained for programming tasks like \"Write a C# function reading a file name from a given URI\" or \"Write some programming interview questions\"."); + "\nIt's a 7B Code Llama, so it's trained for programming tasks like \"Write a C# function reading a file name from a given URI\" or \"Write some programming interview questions\"." + + "\nWrite 'exit' to exit"); Console.ForegroundColor = ConsoleColor.White; var inferenceParams = new InferenceParams() { @@ -43,10 +44,10 @@ MaxTokens = -1, }; - string instruction = $"{SystemInstruction}\n"; + string instruction = $"{SystemInstruction}\n\n"; await Console.Out.WriteAsync("Instruction: "); instruction += Console.ReadLine() ?? "Ask me for instructions."; - while (true) + while (instruction != "exit") { Console.ForegroundColor = ConsoleColor.Green;