From a0743855689f91f5d2f383b8cff91a339ff14628 Mon Sep 17 00:00:00 2001
From: xbotter <xbotter@live.cn>
Date: Sat, 24 Jun 2023 19:59:20 +0800
Subject: [PATCH] add stream example

---
 LLama.WebAPI/Controllers/ChatController.cs   | 15 +++++++++++
 LLama.WebAPI/Services/StatefulChatService.cs | 27 ++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/LLama.WebAPI/Controllers/ChatController.cs b/LLama.WebAPI/Controllers/ChatController.cs
index dab8595d..001a3224 100644
--- a/LLama.WebAPI/Controllers/ChatController.cs
+++ b/LLama.WebAPI/Controllers/ChatController.cs
@@ -23,6 +23,21 @@ namespace LLama.WebAPI.Controllers
             return _service.Send(input);
         }
 
+        [HttpPost("Send/Stream")]
+        public async Task SendMessageStream([FromBody] SendMessageInput input, [FromServices] StatefulChatService _service, CancellationToken cancellationToken)
+        {
+
+            Response.ContentType = "text/event-stream";
+
+            await foreach (var r in _service.SendStream(input))
+            {
+                await Response.WriteAsync("data:" + r + "\n\n", cancellationToken);
+                await Response.Body.FlushAsync(cancellationToken);
+            }
+
+            await Response.CompleteAsync();
+        }
+
         [HttpPost("History")]
         public async Task<string> SendHistory([FromBody] HistoryInput input, [FromServices] StatelessChatService _service)
         {
diff --git a/LLama.WebAPI/Services/StatefulChatService.cs b/LLama.WebAPI/Services/StatefulChatService.cs
index f7afe1a0..ab89b517 100644
--- a/LLama.WebAPI/Services/StatefulChatService.cs
+++ b/LLama.WebAPI/Services/StatefulChatService.cs
@@ -1,6 +1,7 @@
 
 using LLama.WebAPI.Models;
 using Microsoft;
+using System.Runtime.CompilerServices;
 
 namespace LLama.WebAPI.Services;
 
@@ -52,4 +53,30 @@ public class StatefulChatService : IDisposable
 
         return result;
     }
+
+    public async IAsyncEnumerable<string> SendStream(SendMessageInput input)
+    {
+        var userInput = input.Text;
+        if (!_continue)
+        {
+            userInput = SystemPrompt + userInput;
+            Console.Write(SystemPrompt);
+            _continue = true;
+        }
+
+        Console.ForegroundColor = ConsoleColor.Green;
+        Console.Write(input.Text);
+
+        Console.ForegroundColor = ConsoleColor.White;
+        var outputs = _session.ChatAsync(userInput, new Common.InferenceParams()
+        {
+            RepeatPenalty = 1.0f,
+            AntiPrompts = new string[] { "User:" },
+        });
+        await foreach (var output in outputs)
+        {
+            Console.Write(output);
+            yield return output;
+        }
+    }
 }