From ae87dc4a6db86903452ff1264ebea955dd03f502 Mon Sep 17 00:00:00 2001 From: aaishikdutta <107566376+aaishikdutta@users.noreply.github.com> Date: Tue, 11 Jul 2023 01:38:22 +0530 Subject: [PATCH] feat: add streaming support for OpenSource App (#217) --- README.md | 2 +- embedchain/embedchain.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a849db65..1f63ffc4 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ print(naval_chat_bot.chat("what did the author say about happiness?")) ### Stream Response -- You can add config to your query method to stream responses like ChatGPT does. You would require a downstream handler to render the chunk in your desirable format. Currently only supports OpenAI model. +- You can add config to your query method to stream responses like ChatGPT does. You would require a downstream handler to render the chunk in your desirable format. Supports both OpenAI model and OpenSourceApp. - To use this, instantiate a `QueryConfig` or `ChatConfig` object with `stream=True`. Then pass it to the `.chat()` or `.query()` method. The following example iterates through the chunks and prints them as they appear. diff --git a/embedchain/embedchain.py b/embedchain/embedchain.py index d455701c..e219d348 100644 --- a/embedchain/embedchain.py +++ b/embedchain/embedchain.py @@ -350,7 +350,7 @@ class OpenSourceApp(EmbedChain): print("Successfully loaded open source embedding model.") super().__init__(config) - def get_llm_model_answer(self, prompt): + def get_llm_model_answer(self, prompt, config: ChatConfig): from gpt4all import GPT4All global gpt4all_model @@ -358,6 +358,7 @@ class OpenSourceApp(EmbedChain): gpt4all_model = GPT4All("orca-mini-3b.ggmlv3.q4_0.bin") response = gpt4all_model.generate( prompt=prompt, + streaming=config.stream ) return response