From 7aa7a5eeca5cf47917239757685ab7f19b14478f Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Tue, 11 Mar 2025 23:28:22 +0530 Subject: [PATCH 1/9] added the dora-llama-cpp-python node --- node-hub/dora-llama-cpp-python/README.md | 138 ++++++++++++++++++ .../dora_llama_cpp_python/__init__.py | 11 ++ .../dora_llama_cpp_python/__main__.py | 5 + .../dora_llama_cpp_python/main.py | 93 ++++++++++++ node-hub/dora-llama-cpp-python/pyproject.toml | 26 ++++ node-hub/dora-llama-cpp-python/test.yml | 61 ++++++++ .../tests/test_dora_llama_cpp_python.py | 9 ++ 7 files changed, 343 insertions(+) create mode 100644 node-hub/dora-llama-cpp-python/README.md create mode 100644 node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__init__.py create mode 100644 node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__main__.py create mode 100644 node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py create mode 100644 node-hub/dora-llama-cpp-python/pyproject.toml create mode 100644 node-hub/dora-llama-cpp-python/test.yml create mode 100644 node-hub/dora-llama-cpp-python/tests/test_dora_llama_cpp_python.py diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md new file mode 100644 index 00000000..0798c738 --- /dev/null +++ b/node-hub/dora-llama-cpp-python/README.md @@ -0,0 +1,138 @@ +# dora-llama-cpp-python + +A Dora node that provides access to LLaMA-based models using either llama.cpp or Hugging Face backends for text generation. + +## Features + +- Supports both llama.cpp (CPU) and Hugging Face (CPU/GPU) backends +- Easy integration with speech-to-text and text-to-speech pipelines +- Configurable system prompts and activation words +- Chat history support with Hugging Face models +- Lightweight CPU inference with GGUF models + +## Getting started + +### Installation + +```bash +uv venv -p 3.11 --seed +uv pip install -e . +``` + +## Usage + +The node can be configured in your dataflow YAML file: + +```yaml +- id: dora-llama-cpp-python + build: pip install -e path/to/dora-llama-cpp-python + path: dora-llama-cpp-python + inputs: + text: source_node/text # Input text to generate response for + outputs: + - text # Generated response text + env: + MODEL_BACKEND: "llama-cpp" # or "huggingface" + SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." + ACTIVATION_WORDS: "what how who where you" # Space-separated activation words +``` + +### Configuration Options + +- `MODEL_BACKEND`: Choose between: + - `llama-cpp`: Uses GGUF models via llama.cpp (CPU-optimized, default) + - `huggingface`: Uses Hugging Face Transformers models + +- `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior +- `ACTIVATION_WORDS`: Space-separated list of words that trigger model response + +## Examples + +### Basic Speech-to-Text-to-Speech Pipeline + +This example shows how to create a conversational AI pipeline that: +1. Captures audio from microphone +2. Converts speech to text +3. Generates AI responses +4. Converts responses back to speech + +```yaml +nodes: + - id: dora-microphone + build: pip install dora-microphone + path: dora-microphone + inputs: + tick: dora/timer/millis/2000 + outputs: + - audio + + - id: dora-vad + build: pip install dora-vad + path: dora-vad + inputs: + audio: dora-microphone/audio + outputs: + - audio + - timestamp_start + + - id: dora-whisper + build: pip install dora-distil-whisper + path: dora-distil-whisper + inputs: + input: dora-vad/audio + outputs: + - text + + - id: dora-llama-cpp-python + build: pip install -e . + path: dora-llama-cpp-python + inputs: + text: dora-whisper/text + outputs: + - text + env: + MODEL_BACKEND: llama-cpp + SYSTEM_PROMPT: "You're a helpful assistant." + ACTIVATION_WORDS: "hey help what how" + + - id: dora-tts + build: pip install dora-kokoro-tts + path: dora-kokoro-tts + inputs: + text: dora-llama-cpp-python/text + outputs: + - audio +``` + +### Running the Example + +```bash +dora build example.yml +dora run example.yml +``` + +## Contribution Guide + +- Format with [ruff](https://docs.astral.sh/ruff/): + +```bash +uv pip install ruff +uv run ruff check . --fix +``` + +- Lint with ruff: + +```bash +uv run ruff check . +``` + +- Test with [pytest](https://github.com/pytest-dev/pytest) + +```bash +uv pip install pytest +uv run pytest . # Test +``` + +## License + +dora-llama-cpp-python's code is released under the MIT License diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__init__.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__init__.py new file mode 100644 index 00000000..ac3cbef9 --- /dev/null +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__init__.py @@ -0,0 +1,11 @@ +import os + +# Define the path to the README file relative to the package directory +readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md") + +# Read the content of the README file +try: + with open(readme_path, "r", encoding="utf-8") as f: + __doc__ = f.read() +except FileNotFoundError: + __doc__ = "README file not found." diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__main__.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__main__.py new file mode 100644 index 00000000..bcbfde6d --- /dev/null +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__main__.py @@ -0,0 +1,5 @@ +from .main import main + + +if __name__ == "__main__": + main() diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py new file mode 100644 index 00000000..e7a12a10 --- /dev/null +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py @@ -0,0 +1,93 @@ +import os +import pyarrow as pa +from dora import Node +from transformers import AutoModelForCausalLM, AutoTokenizer + +# System prompt +SYSTEM_PROMPT = os.getenv( + "SYSTEM_PROMPT", + "You're a very succinct AI assistant with short answers.", +) + +# Model selection based on ENV variable +MODEL_BACKEND = os.getenv("MODEL_BACKEND", "llama-cpp") # Default to CPU-based Llama + + +def get_model_llama_cpp(): + """Load a GGUF model using llama-cpp-python (CPU by default).""" + from llama_cpp import Llama + + llm = Llama.from_pretrained( + repo_id="Qwen/Qwen2.5-0.5B-Instruct-GGUF", filename="*fp16.gguf", verbose=False + ) + return llm + + +def get_model_huggingface(): + """Load a Hugging Face transformers model.""" + model_name = "Qwen/Qwen2.5-0.5B-Instruct" + + model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="auto", device_map="cpu" + ) + tokenizer = AutoTokenizer.from_pretrained(model_name) + return model, tokenizer + + +ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split() + + +def generate_hf(model, tokenizer, prompt: str, history) -> str: + """Generates text using a Hugging Face model.""" + history += [{"role": "user", "content": prompt}] + text = tokenizer.apply_chat_template( + history, tokenize=False, add_generation_prompt=True + ) + model_inputs = tokenizer([text], return_tensors="pt").to("cpu") + generated_ids = model.generate(**model_inputs, max_new_tokens=512) + generated_ids = [ + output_ids[len(input_ids) :] + for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) + ] + response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] + history += [{"role": "assistant", "content": response}] + return response, history + + +def main(): + history = [] + + # Select model backend + if MODEL_BACKEND == "llama-cpp": + model = get_model_llama_cpp() + else: + model, tokenizer = get_model_huggingface() + + node = Node() + + for event in node: + if event["type"] == "INPUT": + text = event["value"][0].as_py() + words = text.lower().split() + print(words) + + if any(word in ACTIVATION_WORDS for word in words): + print("") + if MODEL_BACKEND == "llama-cpp": + response = model( + f"Q: {text} A: ", + max_tokens=24, + stop=["Q:", "\n"], + )["choices"][0]["text"] + else: + response, history = generate_hf(model, tokenizer, text, history) + + # log output + print(response) + node.send_output( + output_id="text", data=pa.array([response]), metadata={} + ) + + +if __name__ == "__main__": + main() diff --git a/node-hub/dora-llama-cpp-python/pyproject.toml b/node-hub/dora-llama-cpp-python/pyproject.toml new file mode 100644 index 00000000..a5b9c6bb --- /dev/null +++ b/node-hub/dora-llama-cpp-python/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "dora-llama-cpp-python" +version = "0.0.0" +authors = [{ name = "Shashwat Patil", email = "email@email.com" }] +description = "dora-llama-cpp-python" +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.9" + +dependencies = [ + "dora-rs >= 0.3.9", + "torch == 2.4.0", + "torchvision >= 0.19", + "torchaudio >= 2.1.0", + "opencv-python >= 4.1.1", + "modelscope >= 1.18.1", + "accelerate >= 1.3.0", + "transformers", + "llama-cpp-python", +] + +[dependency-groups] +dev = ["pytest >=8.1.1", "ruff >=0.9.1"] + +[project.scripts] +dora-llama-cpp-python = "dora_llama_cpp_python.main:main" diff --git a/node-hub/dora-llama-cpp-python/test.yml b/node-hub/dora-llama-cpp-python/test.yml new file mode 100644 index 00000000..6aa18a97 --- /dev/null +++ b/node-hub/dora-llama-cpp-python/test.yml @@ -0,0 +1,61 @@ +nodes: + - id: dora-microphone + build: pip install -e ../../node-hub/dora-microphone + path: dora-microphone + inputs: + tick: dora/timer/millis/2000 + outputs: + - audio + + - id: dora-vad + build: pip install -e ../../node-hub/dora-vad + path: dora-vad + inputs: + audio: dora-microphone/audio + outputs: + - audio + - timestamp_start + + - id: dora-distil-whisper + build: pip install -e ../../node-hub/dora-distil-whisper + path: dora-distil-whisper + inputs: + input: dora-vad/audio + outputs: + - text + env: + TARGET_LANGUAGE: english + + - id: dora-llama-cpp-python + build: pip install -e ../../node-hub/dora-llama-cpp-python + path: dora-llama-cpp-python + inputs: + text: dora-distil-whisper/text + outputs: + - text + env: + MODEL_BACKEND: llama-cpp # Can be changed to "huggingface" if needed + + - id: plot + build: pip install -e ../../node-hub/dora-rerun + path: dora-rerun + inputs: + text_llama: dora-llama-cpp-python/text + text_whisper: dora-distil-whisper/text + + - id: dora-kokoro-tts + build: pip install -e ../../node-hub/dora-kokoro-tts + path: dora-kokoro-tts + inputs: + text: dora-llama-cpp-python/text + outputs: + - audio + env: + ACTIVATION_WORDS: you + + - id: dora-pyaudio + build: pip install -e ../../node-hub/dora-pyaudio + path: dora-pyaudio + inputs: + audio: dora-kokoro-tts/audio + timestamp_start: dora-vad/timestamp_start diff --git a/node-hub/dora-llama-cpp-python/tests/test_dora_llama_cpp_python.py b/node-hub/dora-llama-cpp-python/tests/test_dora_llama_cpp_python.py new file mode 100644 index 00000000..a4b11758 --- /dev/null +++ b/node-hub/dora-llama-cpp-python/tests/test_dora_llama_cpp_python.py @@ -0,0 +1,9 @@ +import pytest + + +def test_import_main(): + from dora_llama_cpp_python.main import main + + # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow. + with pytest.raises(RuntimeError): + main() From 26e40a2b8b3597e2a7355aa4e978da26eb7ca09e Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Tue, 11 Mar 2025 23:30:26 +0530 Subject: [PATCH 2/9] updated the readme --- node-hub/dora-llama-cpp-python/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md index 0798c738..148bfb3d 100644 --- a/node-hub/dora-llama-cpp-python/README.md +++ b/node-hub/dora-llama-cpp-python/README.md @@ -46,7 +46,7 @@ The node can be configured in your dataflow YAML file: - `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior - `ACTIVATION_WORDS`: Space-separated list of words that trigger model response -## Examples +## Example yml ### Basic Speech-to-Text-to-Speech Pipeline From 49eb2d650369460d30064c5a2b5e3424749297c5 Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Wed, 12 Mar 2025 13:25:41 +0530 Subject: [PATCH 3/9] added model change functionality from yml --- node-hub/dora-llama-cpp-python/README.md | 9 ++++- .../dora_llama_cpp_python/main.py | 34 ++++++++++--------- node-hub/dora-llama-cpp-python/test.yml | 8 ++++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md index 148bfb3d..d107d33c 100644 --- a/node-hub/dora-llama-cpp-python/README.md +++ b/node-hub/dora-llama-cpp-python/README.md @@ -9,6 +9,7 @@ A Dora node that provides access to LLaMA-based models using either llama.cpp or - Configurable system prompts and activation words - Chat history support with Hugging Face models - Lightweight CPU inference with GGUF models +- Multiple model support (Qwen, LLaMA, etc.) ## Getting started @@ -33,8 +34,12 @@ The node can be configured in your dataflow YAML file: - text # Generated response text env: MODEL_BACKEND: "llama-cpp" # or "huggingface" + MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" # For llama-cpp backend + MODEL_FILENAME: "*fp16.gguf" # For llama-cpp backend + HF_MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # For huggingface backend SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." - ACTIVATION_WORDS: "what how who where you" # Space-separated activation words + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "512" ``` ### Configuration Options @@ -92,6 +97,8 @@ nodes: - text env: MODEL_BACKEND: llama-cpp + MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" + MODEL_FILENAME: "*fp16.gguf" SYSTEM_PROMPT: "You're a helpful assistant." ACTIVATION_WORDS: "hey help what how" diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py index e7a12a10..4edc66ae 100644 --- a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py @@ -3,14 +3,16 @@ import pyarrow as pa from dora import Node from transformers import AutoModelForCausalLM, AutoTokenizer -# System prompt +# Environment variables for model configuration SYSTEM_PROMPT = os.getenv( "SYSTEM_PROMPT", "You're a very succinct AI assistant with short answers.", ) - -# Model selection based on ENV variable -MODEL_BACKEND = os.getenv("MODEL_BACKEND", "llama-cpp") # Default to CPU-based Llama +MODEL_BACKEND = os.getenv("MODEL_BACKEND", "llama-cpp") +MODEL_REPO_ID = os.getenv("MODEL_REPO_ID", "Qwen/Qwen2.5-0.5B-Instruct-GGUF") +MODEL_FILENAME = os.getenv("MODEL_FILENAME", "*fp16.gguf") +HF_MODEL_NAME = os.getenv("HF_MODEL_NAME", "Qwen/Qwen2.5-0.5B-Instruct") +MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) def get_model_llama_cpp(): @@ -18,19 +20,21 @@ def get_model_llama_cpp(): from llama_cpp import Llama llm = Llama.from_pretrained( - repo_id="Qwen/Qwen2.5-0.5B-Instruct-GGUF", filename="*fp16.gguf", verbose=False + repo_id=MODEL_REPO_ID, + filename=MODEL_FILENAME, + verbose=False ) return llm def get_model_huggingface(): """Load a Hugging Face transformers model.""" - model_name = "Qwen/Qwen2.5-0.5B-Instruct" - model = AutoModelForCausalLM.from_pretrained( - model_name, torch_dtype="auto", device_map="cpu" + HF_MODEL_NAME, + torch_dtype="auto", + device_map="cpu" ) - tokenizer = AutoTokenizer.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(HF_MODEL_NAME) return model, tokenizer @@ -44,9 +48,9 @@ def generate_hf(model, tokenizer, prompt: str, history) -> str: history, tokenize=False, add_generation_prompt=True ) model_inputs = tokenizer([text], return_tensors="pt").to("cpu") - generated_ids = model.generate(**model_inputs, max_new_tokens=512) + generated_ids = model.generate(**model_inputs, max_new_tokens=MAX_TOKENS) generated_ids = [ - output_ids[len(input_ids) :] + output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) ] response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] @@ -69,21 +73,19 @@ def main(): if event["type"] == "INPUT": text = event["value"][0].as_py() words = text.lower().split() - print(words) + # print(f"Input text: {text}") if any(word in ACTIVATION_WORDS for word in words): - print("") if MODEL_BACKEND == "llama-cpp": response = model( f"Q: {text} A: ", - max_tokens=24, + max_tokens=MAX_TOKENS, stop=["Q:", "\n"], )["choices"][0]["text"] else: response, history = generate_hf(model, tokenizer, text, history) - # log output - print(response) + # print(f"Generated response: {response}") node.send_output( output_id="text", data=pa.array([response]), metadata={} ) diff --git a/node-hub/dora-llama-cpp-python/test.yml b/node-hub/dora-llama-cpp-python/test.yml index 6aa18a97..e34adb4f 100644 --- a/node-hub/dora-llama-cpp-python/test.yml +++ b/node-hub/dora-llama-cpp-python/test.yml @@ -34,7 +34,13 @@ nodes: outputs: - text env: - MODEL_BACKEND: llama-cpp # Can be changed to "huggingface" if needed + MODEL_BACKEND: "llama-cpp" # or "huggingface" + MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" # For llama-cpp backend + MODEL_FILENAME: "*fp16.gguf" # For llama-cpp backend + HF_MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # For huggingface backend + SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "512" - id: plot build: pip install -e ../../node-hub/dora-rerun From 2e0812788133039e5c1c489fdfe84a71711f7478 Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Sat, 15 Mar 2025 16:21:03 +0530 Subject: [PATCH 4/9] updated functionality --- node-hub/dora-llama-cpp-python/README.md | 40 +++++---- .../dora_llama_cpp_python/main.py | 87 +++++++------------ node-hub/dora-llama-cpp-python/pyproject.toml | 19 +++- node-hub/dora-llama-cpp-python/test.yml | 8 +- 4 files changed, 73 insertions(+), 81 deletions(-) diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md index d107d33c..22e4e04c 100644 --- a/node-hub/dora-llama-cpp-python/README.md +++ b/node-hub/dora-llama-cpp-python/README.md @@ -1,15 +1,15 @@ # dora-llama-cpp-python -A Dora node that provides access to LLaMA-based models using either llama.cpp or Hugging Face backends for text generation. +A Dora node that provides access to LLaMA models using llama-cpp-python for efficient CPU/GPU inference. ## Features -- Supports both llama.cpp (CPU) and Hugging Face (CPU/GPU) backends +- GPU acceleration support (CUDA on Linux, Metal on macOS) - Easy integration with speech-to-text and text-to-speech pipelines - Configurable system prompts and activation words -- Chat history support with Hugging Face models - Lightweight CPU inference with GGUF models -- Multiple model support (Qwen, LLaMA, etc.) +- Thread-level CPU optimization +- Adjustable context window size ## Getting started @@ -20,6 +20,7 @@ uv venv -p 3.11 --seed uv pip install -e . ``` + ## Usage The node can be configured in your dataflow YAML file: @@ -33,27 +34,26 @@ The node can be configured in your dataflow YAML file: outputs: - text # Generated response text env: - MODEL_BACKEND: "llama-cpp" # or "huggingface" - MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" # For llama-cpp backend - MODEL_FILENAME: "*fp16.gguf" # For llama-cpp backend - HF_MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # For huggingface backend + MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you" MAX_TOKENS: "512" + N_GPU_LAYERS: "35" # Enable GPU acceleration + N_THREADS: "4" # CPU threads + CONTEXT_SIZE: "4096" # Maximum context window ``` ### Configuration Options -- `MODEL_BACKEND`: Choose between: - - `llama-cpp`: Uses GGUF models via llama.cpp (CPU-optimized, default) - - `huggingface`: Uses Hugging Face Transformers models - +- `MODEL_PATH`: Path to your GGUF model file (default: "./models/llama-2-7b-chat.Q4_K_M.gguf") - `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior - `ACTIVATION_WORDS`: Space-separated list of words that trigger model response +- `MAX_TOKENS`: Maximum number of tokens to generate (default: 512) +- `N_GPU_LAYERS`: Number of layers to offload to GPU (default: 0, set to 35 for GPU acceleration) +- `N_THREADS`: Number of CPU threads to use (default: 4) +- `CONTEXT_SIZE`: Maximum context window size (default: 4096) -## Example yml - -### Basic Speech-to-Text-to-Speech Pipeline +## Example: Speech Assistant Pipeline This example shows how to create a conversational AI pipeline that: 1. Captures audio from microphone @@ -96,11 +96,13 @@ nodes: outputs: - text env: - MODEL_BACKEND: llama-cpp - MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" - MODEL_FILENAME: "*fp16.gguf" + MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" SYSTEM_PROMPT: "You're a helpful assistant." ACTIVATION_WORDS: "hey help what how" + MAX_TOKENS: "512" + N_GPU_LAYERS: "35" + N_THREADS: "4" + CONTEXT_SIZE: "4096" - id: dora-tts build: pip install dora-kokoro-tts @@ -142,4 +144,4 @@ uv run pytest . # Test ## License -dora-llama-cpp-python's code is released under the MIT License +dora-llama-cpp-python is released under the MIT License \ No newline at end of file diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py index 4edc66ae..843a32bf 100644 --- a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py @@ -1,91 +1,64 @@ import os import pyarrow as pa from dora import Node -from transformers import AutoModelForCausalLM, AutoTokenizer +from pathlib import Path # Environment variables for model configuration SYSTEM_PROMPT = os.getenv( "SYSTEM_PROMPT", "You're a very succinct AI assistant with short answers.", ) -MODEL_BACKEND = os.getenv("MODEL_BACKEND", "llama-cpp") -MODEL_REPO_ID = os.getenv("MODEL_REPO_ID", "Qwen/Qwen2.5-0.5B-Instruct-GGUF") -MODEL_FILENAME = os.getenv("MODEL_FILENAME", "*fp16.gguf") -HF_MODEL_NAME = os.getenv("HF_MODEL_NAME", "Qwen/Qwen2.5-0.5B-Instruct") +MODEL_PATH = os.getenv("MODEL_PATH", "./models/llama-2-7b-chat.Q4_K_M.gguf") MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) +N_GPU_LAYERS = int(os.getenv("N_GPU_LAYERS", "0")) # Number of layers to offload to GPU +N_THREADS = int(os.getenv("N_THREADS", "4")) # Number of CPU threads +CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "4096")) -def get_model_llama_cpp(): - """Load a GGUF model using llama-cpp-python (CPU by default).""" +def get_model(): + """Load a GGUF model using llama-cpp-python with optional GPU acceleration.""" from llama_cpp import Llama - - llm = Llama.from_pretrained( - repo_id=MODEL_REPO_ID, - filename=MODEL_FILENAME, + + model_path = Path(MODEL_PATH) + if not model_path.exists(): + raise FileNotFoundError( + f"Model file not found at {MODEL_PATH}. " + "Download it using: wget -O models/llama-2-7b-chat.Q4_K_M.gguf " + "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf" + ) + + llm = Llama( + model_path=str(model_path), + n_gpu_layers=N_GPU_LAYERS, # Enable GPU acceleration if > 0 + n_ctx=CONTEXT_SIZE, # Maximum context size + n_threads=N_THREADS, # Control CPU threading verbose=False ) return llm -def get_model_huggingface(): - """Load a Hugging Face transformers model.""" - model = AutoModelForCausalLM.from_pretrained( - HF_MODEL_NAME, - torch_dtype="auto", - device_map="cpu" - ) - tokenizer = AutoTokenizer.from_pretrained(HF_MODEL_NAME) - return model, tokenizer - - ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split() -def generate_hf(model, tokenizer, prompt: str, history) -> str: - """Generates text using a Hugging Face model.""" - history += [{"role": "user", "content": prompt}] - text = tokenizer.apply_chat_template( - history, tokenize=False, add_generation_prompt=True - ) - model_inputs = tokenizer([text], return_tensors="pt").to("cpu") - generated_ids = model.generate(**model_inputs, max_new_tokens=MAX_TOKENS) - generated_ids = [ - output_ids[len(input_ids):] - for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) - ] - response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] - history += [{"role": "assistant", "content": response}] - return response, history - - def main(): - history = [] - - # Select model backend - if MODEL_BACKEND == "llama-cpp": - model = get_model_llama_cpp() - else: - model, tokenizer = get_model_huggingface() - + # Initialize model + model = get_model() node = Node() for event in node: if event["type"] == "INPUT": text = event["value"][0].as_py() words = text.lower().split() - # print(f"Input text: {text}") if any(word in ACTIVATION_WORDS for word in words): - if MODEL_BACKEND == "llama-cpp": - response = model( - f"Q: {text} A: ", - max_tokens=MAX_TOKENS, - stop=["Q:", "\n"], - )["choices"][0]["text"] - else: - response, history = generate_hf(model, tokenizer, text, history) + # Generate response using system prompt + prompt = f"{SYSTEM_PROMPT}\nQ: {text}\nA:" + response = model( + prompt, + max_tokens=MAX_TOKENS, + stop=["Q:", "\n"], + )["choices"][0]["text"] - # print(f"Generated response: {response}") node.send_output( output_id="text", data=pa.array([response]), metadata={} ) diff --git a/node-hub/dora-llama-cpp-python/pyproject.toml b/node-hub/dora-llama-cpp-python/pyproject.toml index a5b9c6bb..9aa80926 100644 --- a/node-hub/dora-llama-cpp-python/pyproject.toml +++ b/node-hub/dora-llama-cpp-python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dora-llama-cpp-python" -version = "0.0.0" +version = "0.0.1" authors = [{ name = "Shashwat Patil", email = "email@email.com" }] description = "dora-llama-cpp-python" license = { text = "MIT" } @@ -16,9 +16,26 @@ dependencies = [ "modelscope >= 1.18.1", "accelerate >= 1.3.0", "transformers", + "mlx-lm>=0.21.1; sys_platform == 'darwin'", "llama-cpp-python", ] +[tool.uv.sources] +llama-cpp-python = [ + { index = "llama_cpp_python_metal", marker = "sys_platform == 'darwin'" }, + { index = "llama_cpp_python_cu121", marker = "sys_platform == 'linux'" }, +] + +[[tool.uv.index]] +name = "llama_cpp_python_cu121" +url = "https://abetlen.github.io/llama-cpp-python/whl/cu121" +explicit = true + +[[tool.uv.index]] +name = "llama_cpp_python_metal" +url = "https://abetlen.github.io/llama-cpp-python/whl/metal" +explicit = true + [dependency-groups] dev = ["pytest >=8.1.1", "ruff >=0.9.1"] diff --git a/node-hub/dora-llama-cpp-python/test.yml b/node-hub/dora-llama-cpp-python/test.yml index e34adb4f..375dc01e 100644 --- a/node-hub/dora-llama-cpp-python/test.yml +++ b/node-hub/dora-llama-cpp-python/test.yml @@ -34,13 +34,13 @@ nodes: outputs: - text env: - MODEL_BACKEND: "llama-cpp" # or "huggingface" - MODEL_REPO_ID: "Qwen/Qwen2.5-0.5B-Instruct-GGUF" # For llama-cpp backend - MODEL_FILENAME: "*fp16.gguf" # For llama-cpp backend - HF_MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # For huggingface backend + MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you" MAX_TOKENS: "512" + N_GPU_LAYERS: "35" # Enable GPU acceleration + N_THREADS: "4" # CPU threads + CONTEXT_SIZE: "4096" # Maximum context window - id: plot build: pip install -e ../../node-hub/dora-rerun From f7cf3c21758d500ca8e3f11b66d7c98ec373c617 Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Sun, 16 Mar 2025 01:00:20 +0530 Subject: [PATCH 5/9] removed redundant depends --- node-hub/dora-llama-cpp-python/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/node-hub/dora-llama-cpp-python/pyproject.toml b/node-hub/dora-llama-cpp-python/pyproject.toml index 9aa80926..3663bf3b 100644 --- a/node-hub/dora-llama-cpp-python/pyproject.toml +++ b/node-hub/dora-llama-cpp-python/pyproject.toml @@ -14,8 +14,6 @@ dependencies = [ "torchaudio >= 2.1.0", "opencv-python >= 4.1.1", "modelscope >= 1.18.1", - "accelerate >= 1.3.0", - "transformers", "mlx-lm>=0.21.1; sys_platform == 'darwin'", "llama-cpp-python", ] From 84641f93e480fc8af94889a397396c5915285c70 Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Sun, 16 Mar 2025 03:04:17 +0530 Subject: [PATCH 6/9] added support for remote model pulling from huggingface --- node-hub/dora-llama-cpp-python/README.md | 28 ++++++++- .../dora_llama_cpp_python/main.py | 60 +++++++++++++------ node-hub/dora-llama-cpp-python/pyproject.toml | 2 +- node-hub/dora-llama-cpp-python/test.yml | 3 +- 4 files changed, 70 insertions(+), 23 deletions(-) diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md index 22e4e04c..2acac9df 100644 --- a/node-hub/dora-llama-cpp-python/README.md +++ b/node-hub/dora-llama-cpp-python/README.md @@ -26,6 +26,28 @@ uv pip install -e . The node can be configured in your dataflow YAML file: ```yaml + +# Using a local model + +- id: dora-llama-cpp-python + build: pip install -e path/to/dora-llama-cpp-python + path: dora-llama-cpp-python + inputs: + text: source_node/text # Input text to generate response for + outputs: + - text # Generated response text + env: + MODEL_LOCAL_PATH: "./models/my-local-model.gguf" + SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "512" + N_GPU_LAYERS: "35" # Enable GPU acceleration + N_THREADS: "4" # CPU threads + CONTEXT_SIZE: "4096" # Maximum context window + + + +# Using a HuggingFace model - id: dora-llama-cpp-python build: pip install -e path/to/dora-llama-cpp-python path: dora-llama-cpp-python @@ -34,7 +56,8 @@ The node can be configured in your dataflow YAML file: outputs: - text # Generated response text env: - MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" + MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF" + MODEL_FILE_PATTERN: "*Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you" MAX_TOKENS: "512" @@ -96,7 +119,8 @@ nodes: outputs: - text env: - MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" + MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF" + MODEL_FILE_PATTERN: "*Q4_K_M.gguf" SYSTEM_PROMPT: "You're a helpful assistant." ACTIVATION_WORDS: "hey help what how" MAX_TOKENS: "512" diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py index 843a32bf..18016f46 100644 --- a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py @@ -2,16 +2,21 @@ import os import pyarrow as pa from dora import Node from pathlib import Path +import logging +# Configure logging +logging.basicConfig(level=logging.INFO) # Environment variables for model configuration SYSTEM_PROMPT = os.getenv( "SYSTEM_PROMPT", "You're a very succinct AI assistant with short answers.", ) -MODEL_PATH = os.getenv("MODEL_PATH", "./models/llama-2-7b-chat.Q4_K_M.gguf") +MODEL_LOCAL_PATH = os.getenv("MODEL_LOCAL_PATH", "") # Local model path takes precedence +MODEL_NAME = os.getenv("MODEL_NAME", "TheBloke/Llama-2-7B-Chat-GGUF") # HF repo as fallback +MODEL_FILE_PATTERN = os.getenv("MODEL_FILE_PATTERN", "*Q4_K_M.gguf") MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) -N_GPU_LAYERS = int(os.getenv("N_GPU_LAYERS", "0")) # Number of layers to offload to GPU -N_THREADS = int(os.getenv("N_THREADS", "4")) # Number of CPU threads +N_GPU_LAYERS = int(os.getenv("N_GPU_LAYERS", "0")) +N_THREADS = int(os.getenv("N_THREADS", "4")) CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "4096")) @@ -19,22 +24,39 @@ def get_model(): """Load a GGUF model using llama-cpp-python with optional GPU acceleration.""" from llama_cpp import Llama - model_path = Path(MODEL_PATH) - if not model_path.exists(): - raise FileNotFoundError( - f"Model file not found at {MODEL_PATH}. " - "Download it using: wget -O models/llama-2-7b-chat.Q4_K_M.gguf " - "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf" - ) - - llm = Llama( - model_path=str(model_path), - n_gpu_layers=N_GPU_LAYERS, # Enable GPU acceleration if > 0 - n_ctx=CONTEXT_SIZE, # Maximum context size - n_threads=N_THREADS, # Control CPU threading - verbose=False - ) - return llm + try: + # Check if local path is provided + if MODEL_LOCAL_PATH: + model_path = Path(MODEL_LOCAL_PATH) + if not model_path.exists(): + raise FileNotFoundError(f"Local model not found at {MODEL_LOCAL_PATH}") + + logging.info(f"Loading local model from {MODEL_LOCAL_PATH}") + llm = Llama( + model_path=str(model_path), + n_gpu_layers=N_GPU_LAYERS, + n_ctx=CONTEXT_SIZE, + n_threads=N_THREADS, + verbose=False + ) + else: + # Load from HuggingFace if no local path + logging.info(f"Downloading model {MODEL_NAME} with pattern {MODEL_FILE_PATTERN}") + llm = Llama.from_pretrained( + repo_id=MODEL_NAME, + filename=MODEL_FILE_PATTERN, + n_gpu_layers=N_GPU_LAYERS, + n_ctx=CONTEXT_SIZE, + n_threads=N_THREADS, + verbose=False + ) + + logging.info("Model loaded successfully") + return llm + + except Exception as e: + logging.error(f"Error loading model: {e}") + raise ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split() diff --git a/node-hub/dora-llama-cpp-python/pyproject.toml b/node-hub/dora-llama-cpp-python/pyproject.toml index 3663bf3b..a6006fd1 100644 --- a/node-hub/dora-llama-cpp-python/pyproject.toml +++ b/node-hub/dora-llama-cpp-python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dora-llama-cpp-python" -version = "0.0.1" +version = "1.0.0" authors = [{ name = "Shashwat Patil", email = "email@email.com" }] description = "dora-llama-cpp-python" license = { text = "MIT" } diff --git a/node-hub/dora-llama-cpp-python/test.yml b/node-hub/dora-llama-cpp-python/test.yml index 375dc01e..893cbfb4 100644 --- a/node-hub/dora-llama-cpp-python/test.yml +++ b/node-hub/dora-llama-cpp-python/test.yml @@ -34,7 +34,8 @@ nodes: outputs: - text env: - MODEL_PATH: "./models/llama-2-7b-chat.Q4_K_M.gguf" + MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF" # Llama 2.7B model pull from Hugging Face + MODEL_FILE_PATTERN: "*Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you" MAX_TOKENS: "512" From 6f62c363a62aa4d0524b5ebe6392c5b8647bed9c Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Sun, 16 Mar 2025 13:01:12 +0530 Subject: [PATCH 7/9] added the dora transformer node --- node-hub/dora-transformer/README.md | 163 ++++++++++++++++++ .../dora_transformer/__init__.py | 11 ++ .../dora_transformer/__main__.py | 5 + .../dora-transformer/dora_transformer/main.py | 132 ++++++++++++++ node-hub/dora-transformer/pyproject.toml | 26 +++ node-hub/dora-transformer/test.yml | 67 +++++++ .../tests/test_dora_transformer.py | 9 + 7 files changed, 413 insertions(+) create mode 100644 node-hub/dora-transformer/README.md create mode 100644 node-hub/dora-transformer/dora_transformer/__init__.py create mode 100644 node-hub/dora-transformer/dora_transformer/__main__.py create mode 100644 node-hub/dora-transformer/dora_transformer/main.py create mode 100644 node-hub/dora-transformer/pyproject.toml create mode 100644 node-hub/dora-transformer/test.yml create mode 100644 node-hub/dora-transformer/tests/test_dora_transformer.py diff --git a/node-hub/dora-transformer/README.md b/node-hub/dora-transformer/README.md new file mode 100644 index 00000000..f31fce7a --- /dev/null +++ b/node-hub/dora-transformer/README.md @@ -0,0 +1,163 @@ +# dora-transformer + +A Dora node that provides access to Hugging Face transformer models for efficient text generation and chat completion. + +## Features + +- Multi-platform GPU acceleration support (CUDA, CPU, MPS) +- Memory-efficient model loading with 8-bit quantization +- Seamless integration with speech-to-text and text-to-speech pipelines +- Configurable system prompts and activation words +- Support for various transformer models +- Conversation history management +- Optimized for both small and large language models + +## Getting Started + +### Installation + +```bash +uv venv -p 3.11 --seed +uv pip install -e . +``` + +## Usage + +Configure the node in your dataflow YAML file: + +```yaml +- id: dora-transformer + build: pip install -e path/to/dora-transformer + path: dora-transformer + inputs: + text: source_node/text # Input text to generate response for + outputs: + - text # Generated response text + env: + MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # Model from Hugging Face + SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "128" # Reduced for concise responses + DEVICE: "cuda" # Use "cpu" for CPU, "cuda" for NVIDIA GPU, "mps" for Apple Silicon + ENABLE_MEMORY_EFFICIENT: "true" # Enable 8-bit quantization and memory optimizations + TORCH_DTYPE: "float16" # Use half precision for better memory efficiency +``` + +### Configuration Options + +- `MODEL_NAME`: Hugging Face model identifier (default: "Qwen/Qwen2.5-0.5B-Instruct") +- `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior +- `ACTIVATION_WORDS`: Space-separated list of words that trigger model response +- `MAX_TOKENS`: Maximum number of tokens to generate (default: 128) +- `DEVICE`: Computation device to use (default: "cpu") +- `ENABLE_MEMORY_EFFICIENT`: Enable memory optimizations (default: "true") +- `TORCH_DTYPE`: Model precision type (default: "auto") + +### Memory Management + +The node includes several memory optimization features: +- 8-bit quantization for CUDA devices +- Automatic CUDA cache clearing +- Dynamic CPU fallback on OOM errors +- Memory-efficient model loading +- Half-precision support + +## Example: Voice Assistant Pipeline + +Create a conversational AI pipeline that: +1. Captures audio from microphone +2. Converts speech to text +3. Generates AI responses using transformers +4. Converts responses back to speech + +```yaml +nodes: + - id: dora-microphone + build: pip install -e ../../node-hub/dora-microphone + path: dora-microphone + inputs: + tick: dora/timer/millis/2000 + outputs: + - audio + + - id: dora-vad + build: pip install -e ../../node-hub/dora-vad + path: dora-vad + inputs: + audio: dora-microphone/audio + outputs: + - audio + - timestamp_start + + - id: dora-distil-whisper + build: pip install -e ../../node-hub/dora-distil-whisper + path: dora-distil-whisper + inputs: + input: dora-vad/audio + outputs: + - text + env: + TARGET_LANGUAGE: english + + - id: dora-transformer + build: pip install -e ../../node-hub/dora-transformer + path: dora-transformer + inputs: + text: dora-distil-whisper/text + outputs: + - text + env: + MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" + SYSTEM_PROMPT: "You are an AI assistant that gives extremely concise responses, never more than one or two sentences. Always be direct and to the point." + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "128" + DEVICE: "cuda" + ENABLE_MEMORY_EFFICIENT: "true" + TORCH_DTYPE: "float16" + + - id: dora-kokoro-tts + build: pip install -e ../../node-hub/dora-kokoro-tts + path: dora-kokoro-tts + inputs: + text: dora-transformer/text + outputs: + - audio +``` + +### Running the Example + +```bash +dora build example.yml +dora run example.yml +``` + +### Troubleshooting + +If you encounter CUDA out of memory errors: +1. Set `DEVICE: "cpu"` for CPU-only inference +2. Enable memory optimizations with `ENABLE_MEMORY_EFFICIENT: "true"` +3. Use a smaller model or reduce `MAX_TOKENS` +4. Set `TORCH_DTYPE: "float16"` for reduced memory usage + +## Contribution Guide + +Format with [ruff](https://docs.astral.sh/ruff/): +```bash +uv pip install ruff +uv run ruff check . --fix +``` + +Lint with ruff: +```bash +uv run ruff check . +``` + +Test with [pytest](https://github.com/pytest-dev/pytest): +```bash +uv pip install pytest +uv run pytest . # Test +``` + +## License + +dora-transformer is released under the MIT License diff --git a/node-hub/dora-transformer/dora_transformer/__init__.py b/node-hub/dora-transformer/dora_transformer/__init__.py new file mode 100644 index 00000000..ac3cbef9 --- /dev/null +++ b/node-hub/dora-transformer/dora_transformer/__init__.py @@ -0,0 +1,11 @@ +import os + +# Define the path to the README file relative to the package directory +readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md") + +# Read the content of the README file +try: + with open(readme_path, "r", encoding="utf-8") as f: + __doc__ = f.read() +except FileNotFoundError: + __doc__ = "README file not found." diff --git a/node-hub/dora-transformer/dora_transformer/__main__.py b/node-hub/dora-transformer/dora_transformer/__main__.py new file mode 100644 index 00000000..bcbfde6d --- /dev/null +++ b/node-hub/dora-transformer/dora_transformer/__main__.py @@ -0,0 +1,5 @@ +from .main import main + + +if __name__ == "__main__": + main() diff --git a/node-hub/dora-transformer/dora_transformer/main.py b/node-hub/dora-transformer/dora_transformer/main.py new file mode 100644 index 00000000..38abe8a8 --- /dev/null +++ b/node-hub/dora-transformer/dora_transformer/main.py @@ -0,0 +1,132 @@ +import os +import pyarrow as pa +from dora import Node +from transformers import AutoModelForCausalLM, AutoTokenizer +import logging +import torch + +# Configure logging +logging.basicConfig(level=logging.INFO) + +# Environment variables for model configuration +SYSTEM_PROMPT = os.getenv( + "SYSTEM_PROMPT", + "You're a very succinct AI assistant with short answers.", +) +MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-0.5B-Instruct") +MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) +DEVICE = os.getenv("DEVICE", "cpu") +TORCH_DTYPE = os.getenv("TORCH_DTYPE", "auto") +ENABLE_MEMORY_EFFICIENT = os.getenv("ENABLE_MEMORY_EFFICIENT", "true").lower() == "true" + +# Configure PyTorch memory management +if DEVICE == "cuda": + # Set memory efficient settings + os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" + if ENABLE_MEMORY_EFFICIENT: + torch.cuda.empty_cache() + +# Words that trigger the model to respond +ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split() + +def load_model(): + """Load the transformer model and tokenizer.""" + try: + logging.info(f"Loading model {MODEL_NAME} on {DEVICE}") + + # Memory efficient loading + model_kwargs = { + "torch_dtype": TORCH_DTYPE, + "device_map": DEVICE, + } + + if ENABLE_MEMORY_EFFICIENT and DEVICE == "cuda": + model_kwargs.update({ + "low_cpu_mem_usage": True, + "offload_folder": "offload", + "load_in_8bit": True + }) + + model = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, + **model_kwargs + ) + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + logging.info("Model loaded successfully") + return model, tokenizer + except Exception as e: + if "CUDA out of memory" in str(e): + logging.error("CUDA out of memory error. Try:") + logging.error("1. Setting DEVICE=cpu") + logging.error("2. Using a smaller model") + logging.error("3. Setting ENABLE_MEMORY_EFFICIENT=true") + logging.error(f"Error loading model: {e}") + raise + +def generate_response(model, tokenizer, text: str, history) -> tuple[str, list]: + """Generate text using the transformer model.""" + try: + history += [{"role": "user", "content": text}] + prompt = tokenizer.apply_chat_template( + history, tokenize=False, add_generation_prompt=True + ) + + model_inputs = tokenizer([prompt], return_tensors="pt").to(DEVICE) + + with torch.inference_mode(): + generated_ids = model.generate( + **model_inputs, + max_new_tokens=MAX_TOKENS, + pad_token_id=tokenizer.pad_token_id, + do_sample=True, + temperature=0.7, + top_p=0.9, + repetition_penalty=1.2, # Reduce repetition + length_penalty=0.5, + ) + + generated_ids = [ + output_ids[len(input_ids):] + for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) + ] + + response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] + history += [{"role": "assistant", "content": response}] + + # Clear CUDA cache after generation if enabled + if ENABLE_MEMORY_EFFICIENT and DEVICE == "cuda": + torch.cuda.empty_cache() + + return response, history + except RuntimeError as e: + if "CUDA out of memory" in str(e): + logging.error("CUDA out of memory during generation. Falling back to CPU") + model.to("cpu") + return generate_response(model, tokenizer, text, history) + raise + +def main(): + # Initialize model and conversation history + model, tokenizer = load_model() + # Initialize history with system prompt + history = [{"role": "system", "content": SYSTEM_PROMPT}] + node = Node() + + for event in node: + if event["type"] == "INPUT": + text = event["value"][0].as_py() + words = text.lower().split() + + if any(word in ACTIVATION_WORDS for word in words): + logging.info(f"Processing input: {text}") + response, history = generate_response(model, tokenizer, text, history) + logging.info(f"Generated response: {response}") + + node.send_output( + output_id="text", + data=pa.array([response]), + metadata={} + ) + +if __name__ == "__main__": + main() diff --git a/node-hub/dora-transformer/pyproject.toml b/node-hub/dora-transformer/pyproject.toml new file mode 100644 index 00000000..22756cd6 --- /dev/null +++ b/node-hub/dora-transformer/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "dora-transformer" +version = "0.0.0" +authors = [{ name = "Shashwat Patil", email = "email@email.com" }] +description = "dora-transformer" +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.9" + +dependencies = [ + "dora-rs >= 0.3.9", + "torch == 2.4.0", + "torchvision >= 0.19", + "torchaudio >= 2.1.0", + "opencv-python >= 4.1.1", + "modelscope >= 1.18.1", + "accelerate >= 1.3.0", + "transformers", + "bitsandbytes>=0.41.1", +] + +[dependency-groups] +dev = ["pytest >=8.1.1", "ruff >=0.9.1"] + +[project.scripts] +dora-transformer = "dora_transformer.main:main" diff --git a/node-hub/dora-transformer/test.yml b/node-hub/dora-transformer/test.yml new file mode 100644 index 00000000..30ded08a --- /dev/null +++ b/node-hub/dora-transformer/test.yml @@ -0,0 +1,67 @@ +nodes: + - id: dora-microphone + build: pip install -e ../../node-hub/dora-microphone + path: dora-microphone + inputs: + tick: dora/timer/millis/2000 + outputs: + - audio + + - id: dora-vad + build: pip install -e ../../node-hub/dora-vad + path: dora-vad + inputs: + audio: dora-microphone/audio + outputs: + - audio + - timestamp_start + + - id: dora-distil-whisper + build: pip install -e ../../node-hub/dora-distil-whisper + path: dora-distil-whisper + inputs: + input: dora-vad/audio + outputs: + - text + env: + TARGET_LANGUAGE: english + + - id: dora-transformer + build: pip install -e ../../node-hub/dora-transformer + path: dora-transformer + inputs: + text: dora-distil-whisper/text + outputs: + - text + env: + MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" + SYSTEM_PROMPT: "You are an AI assistant that gives extremely concise responses, never more than one or two sentences. Always be direct and to the point." + ACTIVATION_WORDS: "what how who where you" + MAX_TOKENS: "128" + DEVICE: "cuda" # Change to cpu/mps as needed + ENABLE_MEMORY_EFFICIENT: "true" + TORCH_DTYPE: "float16" + + - id: plot + build: pip install -e ../../node-hub/dora-rerun + path: dora-rerun + inputs: + text_transformer: dora-transformer/text + text_whisper: dora-distil-whisper/text + + - id: dora-kokoro-tts + build: pip install -e ../../node-hub/dora-kokoro-tts + path: dora-kokoro-tts + inputs: + text: dora-transformer/text + outputs: + - audio + env: + ACTIVATION_WORDS: you + + - id: dora-pyaudio + build: pip install -e ../../node-hub/dora-pyaudio + path: dora-pyaudio + inputs: + audio: dora-kokoro-tts/audio + timestamp_start: dora-vad/timestamp_start \ No newline at end of file diff --git a/node-hub/dora-transformer/tests/test_dora_transformer.py b/node-hub/dora-transformer/tests/test_dora_transformer.py new file mode 100644 index 00000000..4f904ff8 --- /dev/null +++ b/node-hub/dora-transformer/tests/test_dora_transformer.py @@ -0,0 +1,9 @@ +import pytest + + +def test_import_main(): + from dora_transformer.main import main + + # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow. + with pytest.raises(RuntimeError): + main() From 650b8837b43939b5930b2dc2373b65dfe5b5619f Mon Sep 17 00:00:00 2001 From: ShashwatPatil Date: Sun, 16 Mar 2025 13:06:35 +0530 Subject: [PATCH 8/9] Revert "added the dora transformer node" This reverts commit 6f62c363a62aa4d0524b5ebe6392c5b8647bed9c. --- node-hub/dora-transformer/README.md | 163 ------------------ .../dora_transformer/__init__.py | 11 -- .../dora_transformer/__main__.py | 5 - .../dora-transformer/dora_transformer/main.py | 132 -------------- node-hub/dora-transformer/pyproject.toml | 26 --- node-hub/dora-transformer/test.yml | 67 ------- .../tests/test_dora_transformer.py | 9 - 7 files changed, 413 deletions(-) delete mode 100644 node-hub/dora-transformer/README.md delete mode 100644 node-hub/dora-transformer/dora_transformer/__init__.py delete mode 100644 node-hub/dora-transformer/dora_transformer/__main__.py delete mode 100644 node-hub/dora-transformer/dora_transformer/main.py delete mode 100644 node-hub/dora-transformer/pyproject.toml delete mode 100644 node-hub/dora-transformer/test.yml delete mode 100644 node-hub/dora-transformer/tests/test_dora_transformer.py diff --git a/node-hub/dora-transformer/README.md b/node-hub/dora-transformer/README.md deleted file mode 100644 index f31fce7a..00000000 --- a/node-hub/dora-transformer/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# dora-transformer - -A Dora node that provides access to Hugging Face transformer models for efficient text generation and chat completion. - -## Features - -- Multi-platform GPU acceleration support (CUDA, CPU, MPS) -- Memory-efficient model loading with 8-bit quantization -- Seamless integration with speech-to-text and text-to-speech pipelines -- Configurable system prompts and activation words -- Support for various transformer models -- Conversation history management -- Optimized for both small and large language models - -## Getting Started - -### Installation - -```bash -uv venv -p 3.11 --seed -uv pip install -e . -``` - -## Usage - -Configure the node in your dataflow YAML file: - -```yaml -- id: dora-transformer - build: pip install -e path/to/dora-transformer - path: dora-transformer - inputs: - text: source_node/text # Input text to generate response for - outputs: - - text # Generated response text - env: - MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" # Model from Hugging Face - SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." - ACTIVATION_WORDS: "what how who where you" - MAX_TOKENS: "128" # Reduced for concise responses - DEVICE: "cuda" # Use "cpu" for CPU, "cuda" for NVIDIA GPU, "mps" for Apple Silicon - ENABLE_MEMORY_EFFICIENT: "true" # Enable 8-bit quantization and memory optimizations - TORCH_DTYPE: "float16" # Use half precision for better memory efficiency -``` - -### Configuration Options - -- `MODEL_NAME`: Hugging Face model identifier (default: "Qwen/Qwen2.5-0.5B-Instruct") -- `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior -- `ACTIVATION_WORDS`: Space-separated list of words that trigger model response -- `MAX_TOKENS`: Maximum number of tokens to generate (default: 128) -- `DEVICE`: Computation device to use (default: "cpu") -- `ENABLE_MEMORY_EFFICIENT`: Enable memory optimizations (default: "true") -- `TORCH_DTYPE`: Model precision type (default: "auto") - -### Memory Management - -The node includes several memory optimization features: -- 8-bit quantization for CUDA devices -- Automatic CUDA cache clearing -- Dynamic CPU fallback on OOM errors -- Memory-efficient model loading -- Half-precision support - -## Example: Voice Assistant Pipeline - -Create a conversational AI pipeline that: -1. Captures audio from microphone -2. Converts speech to text -3. Generates AI responses using transformers -4. Converts responses back to speech - -```yaml -nodes: - - id: dora-microphone - build: pip install -e ../../node-hub/dora-microphone - path: dora-microphone - inputs: - tick: dora/timer/millis/2000 - outputs: - - audio - - - id: dora-vad - build: pip install -e ../../node-hub/dora-vad - path: dora-vad - inputs: - audio: dora-microphone/audio - outputs: - - audio - - timestamp_start - - - id: dora-distil-whisper - build: pip install -e ../../node-hub/dora-distil-whisper - path: dora-distil-whisper - inputs: - input: dora-vad/audio - outputs: - - text - env: - TARGET_LANGUAGE: english - - - id: dora-transformer - build: pip install -e ../../node-hub/dora-transformer - path: dora-transformer - inputs: - text: dora-distil-whisper/text - outputs: - - text - env: - MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" - SYSTEM_PROMPT: "You are an AI assistant that gives extremely concise responses, never more than one or two sentences. Always be direct and to the point." - ACTIVATION_WORDS: "what how who where you" - MAX_TOKENS: "128" - DEVICE: "cuda" - ENABLE_MEMORY_EFFICIENT: "true" - TORCH_DTYPE: "float16" - - - id: dora-kokoro-tts - build: pip install -e ../../node-hub/dora-kokoro-tts - path: dora-kokoro-tts - inputs: - text: dora-transformer/text - outputs: - - audio -``` - -### Running the Example - -```bash -dora build example.yml -dora run example.yml -``` - -### Troubleshooting - -If you encounter CUDA out of memory errors: -1. Set `DEVICE: "cpu"` for CPU-only inference -2. Enable memory optimizations with `ENABLE_MEMORY_EFFICIENT: "true"` -3. Use a smaller model or reduce `MAX_TOKENS` -4. Set `TORCH_DTYPE: "float16"` for reduced memory usage - -## Contribution Guide - -Format with [ruff](https://docs.astral.sh/ruff/): -```bash -uv pip install ruff -uv run ruff check . --fix -``` - -Lint with ruff: -```bash -uv run ruff check . -``` - -Test with [pytest](https://github.com/pytest-dev/pytest): -```bash -uv pip install pytest -uv run pytest . # Test -``` - -## License - -dora-transformer is released under the MIT License diff --git a/node-hub/dora-transformer/dora_transformer/__init__.py b/node-hub/dora-transformer/dora_transformer/__init__.py deleted file mode 100644 index ac3cbef9..00000000 --- a/node-hub/dora-transformer/dora_transformer/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -import os - -# Define the path to the README file relative to the package directory -readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md") - -# Read the content of the README file -try: - with open(readme_path, "r", encoding="utf-8") as f: - __doc__ = f.read() -except FileNotFoundError: - __doc__ = "README file not found." diff --git a/node-hub/dora-transformer/dora_transformer/__main__.py b/node-hub/dora-transformer/dora_transformer/__main__.py deleted file mode 100644 index bcbfde6d..00000000 --- a/node-hub/dora-transformer/dora_transformer/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .main import main - - -if __name__ == "__main__": - main() diff --git a/node-hub/dora-transformer/dora_transformer/main.py b/node-hub/dora-transformer/dora_transformer/main.py deleted file mode 100644 index 38abe8a8..00000000 --- a/node-hub/dora-transformer/dora_transformer/main.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import pyarrow as pa -from dora import Node -from transformers import AutoModelForCausalLM, AutoTokenizer -import logging -import torch - -# Configure logging -logging.basicConfig(level=logging.INFO) - -# Environment variables for model configuration -SYSTEM_PROMPT = os.getenv( - "SYSTEM_PROMPT", - "You're a very succinct AI assistant with short answers.", -) -MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-0.5B-Instruct") -MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) -DEVICE = os.getenv("DEVICE", "cpu") -TORCH_DTYPE = os.getenv("TORCH_DTYPE", "auto") -ENABLE_MEMORY_EFFICIENT = os.getenv("ENABLE_MEMORY_EFFICIENT", "true").lower() == "true" - -# Configure PyTorch memory management -if DEVICE == "cuda": - # Set memory efficient settings - os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" - if ENABLE_MEMORY_EFFICIENT: - torch.cuda.empty_cache() - -# Words that trigger the model to respond -ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split() - -def load_model(): - """Load the transformer model and tokenizer.""" - try: - logging.info(f"Loading model {MODEL_NAME} on {DEVICE}") - - # Memory efficient loading - model_kwargs = { - "torch_dtype": TORCH_DTYPE, - "device_map": DEVICE, - } - - if ENABLE_MEMORY_EFFICIENT and DEVICE == "cuda": - model_kwargs.update({ - "low_cpu_mem_usage": True, - "offload_folder": "offload", - "load_in_8bit": True - }) - - model = AutoModelForCausalLM.from_pretrained( - MODEL_NAME, - **model_kwargs - ) - tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) - logging.info("Model loaded successfully") - return model, tokenizer - except Exception as e: - if "CUDA out of memory" in str(e): - logging.error("CUDA out of memory error. Try:") - logging.error("1. Setting DEVICE=cpu") - logging.error("2. Using a smaller model") - logging.error("3. Setting ENABLE_MEMORY_EFFICIENT=true") - logging.error(f"Error loading model: {e}") - raise - -def generate_response(model, tokenizer, text: str, history) -> tuple[str, list]: - """Generate text using the transformer model.""" - try: - history += [{"role": "user", "content": text}] - prompt = tokenizer.apply_chat_template( - history, tokenize=False, add_generation_prompt=True - ) - - model_inputs = tokenizer([prompt], return_tensors="pt").to(DEVICE) - - with torch.inference_mode(): - generated_ids = model.generate( - **model_inputs, - max_new_tokens=MAX_TOKENS, - pad_token_id=tokenizer.pad_token_id, - do_sample=True, - temperature=0.7, - top_p=0.9, - repetition_penalty=1.2, # Reduce repetition - length_penalty=0.5, - ) - - generated_ids = [ - output_ids[len(input_ids):] - for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) - ] - - response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] - history += [{"role": "assistant", "content": response}] - - # Clear CUDA cache after generation if enabled - if ENABLE_MEMORY_EFFICIENT and DEVICE == "cuda": - torch.cuda.empty_cache() - - return response, history - except RuntimeError as e: - if "CUDA out of memory" in str(e): - logging.error("CUDA out of memory during generation. Falling back to CPU") - model.to("cpu") - return generate_response(model, tokenizer, text, history) - raise - -def main(): - # Initialize model and conversation history - model, tokenizer = load_model() - # Initialize history with system prompt - history = [{"role": "system", "content": SYSTEM_PROMPT}] - node = Node() - - for event in node: - if event["type"] == "INPUT": - text = event["value"][0].as_py() - words = text.lower().split() - - if any(word in ACTIVATION_WORDS for word in words): - logging.info(f"Processing input: {text}") - response, history = generate_response(model, tokenizer, text, history) - logging.info(f"Generated response: {response}") - - node.send_output( - output_id="text", - data=pa.array([response]), - metadata={} - ) - -if __name__ == "__main__": - main() diff --git a/node-hub/dora-transformer/pyproject.toml b/node-hub/dora-transformer/pyproject.toml deleted file mode 100644 index 22756cd6..00000000 --- a/node-hub/dora-transformer/pyproject.toml +++ /dev/null @@ -1,26 +0,0 @@ -[project] -name = "dora-transformer" -version = "0.0.0" -authors = [{ name = "Shashwat Patil", email = "email@email.com" }] -description = "dora-transformer" -license = { text = "MIT" } -readme = "README.md" -requires-python = ">=3.9" - -dependencies = [ - "dora-rs >= 0.3.9", - "torch == 2.4.0", - "torchvision >= 0.19", - "torchaudio >= 2.1.0", - "opencv-python >= 4.1.1", - "modelscope >= 1.18.1", - "accelerate >= 1.3.0", - "transformers", - "bitsandbytes>=0.41.1", -] - -[dependency-groups] -dev = ["pytest >=8.1.1", "ruff >=0.9.1"] - -[project.scripts] -dora-transformer = "dora_transformer.main:main" diff --git a/node-hub/dora-transformer/test.yml b/node-hub/dora-transformer/test.yml deleted file mode 100644 index 30ded08a..00000000 --- a/node-hub/dora-transformer/test.yml +++ /dev/null @@ -1,67 +0,0 @@ -nodes: - - id: dora-microphone - build: pip install -e ../../node-hub/dora-microphone - path: dora-microphone - inputs: - tick: dora/timer/millis/2000 - outputs: - - audio - - - id: dora-vad - build: pip install -e ../../node-hub/dora-vad - path: dora-vad - inputs: - audio: dora-microphone/audio - outputs: - - audio - - timestamp_start - - - id: dora-distil-whisper - build: pip install -e ../../node-hub/dora-distil-whisper - path: dora-distil-whisper - inputs: - input: dora-vad/audio - outputs: - - text - env: - TARGET_LANGUAGE: english - - - id: dora-transformer - build: pip install -e ../../node-hub/dora-transformer - path: dora-transformer - inputs: - text: dora-distil-whisper/text - outputs: - - text - env: - MODEL_NAME: "Qwen/Qwen2.5-0.5B-Instruct" - SYSTEM_PROMPT: "You are an AI assistant that gives extremely concise responses, never more than one or two sentences. Always be direct and to the point." - ACTIVATION_WORDS: "what how who where you" - MAX_TOKENS: "128" - DEVICE: "cuda" # Change to cpu/mps as needed - ENABLE_MEMORY_EFFICIENT: "true" - TORCH_DTYPE: "float16" - - - id: plot - build: pip install -e ../../node-hub/dora-rerun - path: dora-rerun - inputs: - text_transformer: dora-transformer/text - text_whisper: dora-distil-whisper/text - - - id: dora-kokoro-tts - build: pip install -e ../../node-hub/dora-kokoro-tts - path: dora-kokoro-tts - inputs: - text: dora-transformer/text - outputs: - - audio - env: - ACTIVATION_WORDS: you - - - id: dora-pyaudio - build: pip install -e ../../node-hub/dora-pyaudio - path: dora-pyaudio - inputs: - audio: dora-kokoro-tts/audio - timestamp_start: dora-vad/timestamp_start \ No newline at end of file diff --git a/node-hub/dora-transformer/tests/test_dora_transformer.py b/node-hub/dora-transformer/tests/test_dora_transformer.py deleted file mode 100644 index 4f904ff8..00000000 --- a/node-hub/dora-transformer/tests/test_dora_transformer.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - - -def test_import_main(): - from dora_transformer.main import main - - # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow. - with pytest.raises(RuntimeError): - main() From 5361cb930f3d09fe2226cb63164e2f2a5f81ec7a Mon Sep 17 00:00:00 2001 From: Shashwat Patil <117521627+ShashwatPatil@users.noreply.github.com> Date: Sun, 16 Mar 2025 15:56:21 +0530 Subject: [PATCH 9/9] updated theparameter input --- node-hub/dora-llama-cpp-python/README.md | 25 +++---------------- .../dora_llama_cpp_python/main.py | 20 ++++++--------- node-hub/dora-llama-cpp-python/pyproject.toml | 2 +- node-hub/dora-llama-cpp-python/test.yml | 2 +- 4 files changed, 13 insertions(+), 36 deletions(-) diff --git a/node-hub/dora-llama-cpp-python/README.md b/node-hub/dora-llama-cpp-python/README.md index 2acac9df..0ee13479 100644 --- a/node-hub/dora-llama-cpp-python/README.md +++ b/node-hub/dora-llama-cpp-python/README.md @@ -27,26 +27,6 @@ The node can be configured in your dataflow YAML file: ```yaml -# Using a local model - -- id: dora-llama-cpp-python - build: pip install -e path/to/dora-llama-cpp-python - path: dora-llama-cpp-python - inputs: - text: source_node/text # Input text to generate response for - outputs: - - text # Generated response text - env: - MODEL_LOCAL_PATH: "./models/my-local-model.gguf" - SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." - ACTIVATION_WORDS: "what how who where you" - MAX_TOKENS: "512" - N_GPU_LAYERS: "35" # Enable GPU acceleration - N_THREADS: "4" # CPU threads - CONTEXT_SIZE: "4096" # Maximum context window - - - # Using a HuggingFace model - id: dora-llama-cpp-python build: pip install -e path/to/dora-llama-cpp-python @@ -56,7 +36,7 @@ The node can be configured in your dataflow YAML file: outputs: - text # Generated response text env: - MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF" + MODEL_NAME_OR_PATH: "TheBloke/Llama-2-7B-Chat-GGUF" MODEL_FILE_PATTERN: "*Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you" @@ -68,7 +48,8 @@ The node can be configured in your dataflow YAML file: ### Configuration Options -- `MODEL_PATH`: Path to your GGUF model file (default: "./models/llama-2-7b-chat.Q4_K_M.gguf") +- `MODEL_NAME_OR_PATH`: Path to local model file or HuggingFace repo id (default: "TheBloke/Llama-2-7B-Chat-GGUF") +- `MODEL_FILE_PATTERN`: Pattern to match model file when downloading from HF (default: "*Q4_K_M.gguf") - `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior - `ACTIVATION_WORDS`: Space-separated list of words that trigger model response - `MAX_TOKENS`: Maximum number of tokens to generate (default: 512) diff --git a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py index 18016f46..05a7bd24 100644 --- a/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py +++ b/node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py @@ -11,8 +11,7 @@ SYSTEM_PROMPT = os.getenv( "SYSTEM_PROMPT", "You're a very succinct AI assistant with short answers.", ) -MODEL_LOCAL_PATH = os.getenv("MODEL_LOCAL_PATH", "") # Local model path takes precedence -MODEL_NAME = os.getenv("MODEL_NAME", "TheBloke/Llama-2-7B-Chat-GGUF") # HF repo as fallback +MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", "TheBloke/Llama-2-7B-Chat-GGUF") MODEL_FILE_PATTERN = os.getenv("MODEL_FILE_PATTERN", "*Q4_K_M.gguf") MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512")) N_GPU_LAYERS = int(os.getenv("N_GPU_LAYERS", "0")) @@ -25,13 +24,10 @@ def get_model(): from llama_cpp import Llama try: - # Check if local path is provided - if MODEL_LOCAL_PATH: - model_path = Path(MODEL_LOCAL_PATH) - if not model_path.exists(): - raise FileNotFoundError(f"Local model not found at {MODEL_LOCAL_PATH}") - - logging.info(f"Loading local model from {MODEL_LOCAL_PATH}") + # Check if path exists locally + model_path = Path(MODEL_NAME_OR_PATH) + if model_path.exists(): + logging.info(f"Loading local model from {MODEL_NAME_OR_PATH}") llm = Llama( model_path=str(model_path), n_gpu_layers=N_GPU_LAYERS, @@ -40,10 +36,10 @@ def get_model(): verbose=False ) else: - # Load from HuggingFace if no local path - logging.info(f"Downloading model {MODEL_NAME} with pattern {MODEL_FILE_PATTERN}") + # Load from HuggingFace + logging.info(f"Downloading model {MODEL_NAME_OR_PATH} with pattern {MODEL_FILE_PATTERN}") llm = Llama.from_pretrained( - repo_id=MODEL_NAME, + repo_id=MODEL_NAME_OR_PATH, filename=MODEL_FILE_PATTERN, n_gpu_layers=N_GPU_LAYERS, n_ctx=CONTEXT_SIZE, diff --git a/node-hub/dora-llama-cpp-python/pyproject.toml b/node-hub/dora-llama-cpp-python/pyproject.toml index a6006fd1..8b832f72 100644 --- a/node-hub/dora-llama-cpp-python/pyproject.toml +++ b/node-hub/dora-llama-cpp-python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dora-llama-cpp-python" -version = "1.0.0" +version = "1.0.1" authors = [{ name = "Shashwat Patil", email = "email@email.com" }] description = "dora-llama-cpp-python" license = { text = "MIT" } diff --git a/node-hub/dora-llama-cpp-python/test.yml b/node-hub/dora-llama-cpp-python/test.yml index 893cbfb4..3a5a210f 100644 --- a/node-hub/dora-llama-cpp-python/test.yml +++ b/node-hub/dora-llama-cpp-python/test.yml @@ -34,7 +34,7 @@ nodes: outputs: - text env: - MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF" # Llama 2.7B model pull from Hugging Face + MODEL_NAME_OR_PATH: TheBloke/Llama-2-7B-Chat-GGUF # Llama 2.7B model pull from Hugging Face MODEL_FILE_PATTERN: "*Q4_K_M.gguf" SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers." ACTIVATION_WORDS: "what how who where you"