Added the dora-llama-cpp-python node (#850)

This pull request introduces a new Dora node, `dora-llama-cpp-python`,
which provides access to LLaMA-based models using either llama.cpp or
Hugging Face backends for text generation.

node-hub/dora-llama-cpp-python/README.md

@@ -0,0 +1,152 @@
+# dora-llama-cpp-python
+
+A Dora node that provides access to LLaMA models using llama-cpp-python for efficient CPU/GPU inference.
+
+## Features
+
+- GPU acceleration support (CUDA on Linux, Metal on macOS)
+- Easy integration with speech-to-text and text-to-speech pipelines  
+- Configurable system prompts and activation words
+- Lightweight CPU inference with GGUF models
+- Thread-level CPU optimization
+- Adjustable context window size
+
+## Getting started
+
+### Installation
+
+```bash
+uv venv -p 3.11 --seed
+uv pip install -e .
+```
+
+
+## Usage
+
+The node can be configured in your dataflow YAML file:
+
+```yaml
+
+# Using a HuggingFace model
+- id: dora-llama-cpp-python
+  build: pip install -e path/to/dora-llama-cpp-python
+  path: dora-llama-cpp-python
+  inputs:
+    text: source_node/text  # Input text to generate response for
+  outputs:
+    - text  # Generated response text
+  env:
+    MODEL_NAME_OR_PATH: "TheBloke/Llama-2-7B-Chat-GGUF"
+    MODEL_FILE_PATTERN: "*Q4_K_M.gguf"
+    SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers."
+    ACTIVATION_WORDS: "what how who where you"
+    MAX_TOKENS: "512"
+    N_GPU_LAYERS: "35"     # Enable GPU acceleration
+    N_THREADS: "4"         # CPU threads
+    CONTEXT_SIZE: "4096"   # Maximum context window
+```
+
+### Configuration Options
+
+- `MODEL_NAME_OR_PATH`: Path to local model file or HuggingFace repo id (default: "TheBloke/Llama-2-7B-Chat-GGUF")
+- `MODEL_FILE_PATTERN`: Pattern to match model file when downloading from HF (default: "*Q4_K_M.gguf")
+- `SYSTEM_PROMPT`: Customize the AI assistant's personality/behavior
+- `ACTIVATION_WORDS`: Space-separated list of words that trigger model response
+- `MAX_TOKENS`: Maximum number of tokens to generate (default: 512)
+- `N_GPU_LAYERS`: Number of layers to offload to GPU (default: 0, set to 35 for GPU acceleration)
+- `N_THREADS`: Number of CPU threads to use (default: 4)
+- `CONTEXT_SIZE`: Maximum context window size (default: 4096)
+
+## Example: Speech Assistant Pipeline
+
+This example shows how to create a conversational AI pipeline that:
+1. Captures audio from microphone
+2. Converts speech to text
+3. Generates AI responses
+4. Converts responses back to speech
+
+```yaml
+nodes:
+  - id: dora-microphone
+    build: pip install dora-microphone
+    path: dora-microphone
+    inputs:
+      tick: dora/timer/millis/2000
+    outputs:
+      - audio
+
+  - id: dora-vad
+    build: pip install dora-vad
+    path: dora-vad
+    inputs:
+      audio: dora-microphone/audio
+    outputs:
+      - audio
+      - timestamp_start
+
+  - id: dora-whisper
+    build: pip install dora-distil-whisper
+    path: dora-distil-whisper
+    inputs:
+      input: dora-vad/audio
+    outputs:
+      - text
+
+  - id: dora-llama-cpp-python
+    build: pip install -e .
+    path: dora-llama-cpp-python
+    inputs:
+      text: dora-whisper/text
+    outputs:
+      - text
+    env:
+      MODEL_NAME: "TheBloke/Llama-2-7B-Chat-GGUF"
+      MODEL_FILE_PATTERN: "*Q4_K_M.gguf"
+      SYSTEM_PROMPT: "You're a helpful assistant."
+      ACTIVATION_WORDS: "hey help what how"
+      MAX_TOKENS: "512"
+      N_GPU_LAYERS: "35"
+      N_THREADS: "4"
+      CONTEXT_SIZE: "4096"
+
+  - id: dora-tts
+    build: pip install dora-kokoro-tts
+    path: dora-kokoro-tts
+    inputs:
+      text: dora-llama-cpp-python/text
+    outputs:
+      - audio
+```
+
+### Running the Example
+
+```bash
+dora build example.yml
+dora run example.yml
+```
+
+## Contribution Guide
+
+- Format with [ruff](https://docs.astral.sh/ruff/):
+
+```bash
+uv pip install ruff
+uv run ruff check . --fix
+```
+
+- Lint with ruff:
+
+```bash
+uv run ruff check .
+```
+
+- Test with [pytest](https://github.com/pytest-dev/pytest)
+
+```bash
+uv pip install pytest
+uv run pytest . # Test
+```
+
+## License
+
+dora-llama-cpp-python is released under the MIT License

node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+    with open(readme_path, "r", encoding="utf-8") as f:
+        __doc__ = f.read()
+except FileNotFoundError:
+    __doc__ = "README file not found."

node-hub/dora-llama-cpp-python/dora_llama_cpp_python/__main__.py

@@ -0,0 +1,5 @@
+from .main import main
+
+
+if __name__ == "__main__":
+    main()

node-hub/dora-llama-cpp-python/dora_llama_cpp_python/main.py

@@ -0,0 +1,86 @@
+import os
+import pyarrow as pa
+from dora import Node
+from pathlib import Path
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+# Environment variables for model configuration
+SYSTEM_PROMPT = os.getenv(
+    "SYSTEM_PROMPT",
+    "You're a very succinct AI assistant with short answers.",
+)
+MODEL_NAME_OR_PATH = os.getenv("MODEL_NAME_OR_PATH", "TheBloke/Llama-2-7B-Chat-GGUF")
+MODEL_FILE_PATTERN = os.getenv("MODEL_FILE_PATTERN", "*Q4_K_M.gguf")
+MAX_TOKENS = int(os.getenv("MAX_TOKENS", "512"))
+N_GPU_LAYERS = int(os.getenv("N_GPU_LAYERS", "0"))
+N_THREADS = int(os.getenv("N_THREADS", "4"))
+CONTEXT_SIZE = int(os.getenv("CONTEXT_SIZE", "4096"))
+
+
+def get_model():
+    """Load a GGUF model using llama-cpp-python with optional GPU acceleration."""
+    from llama_cpp import Llama
+    
+    try:
+        # Check if path exists locally
+        model_path = Path(MODEL_NAME_OR_PATH)
+        if model_path.exists():
+            logging.info(f"Loading local model from {MODEL_NAME_OR_PATH}")
+            llm = Llama(
+                model_path=str(model_path),
+                n_gpu_layers=N_GPU_LAYERS,
+                n_ctx=CONTEXT_SIZE,
+                n_threads=N_THREADS,
+                verbose=False
+            )
+        else:
+            # Load from HuggingFace
+            logging.info(f"Downloading model {MODEL_NAME_OR_PATH} with pattern {MODEL_FILE_PATTERN}")
+            llm = Llama.from_pretrained(
+                repo_id=MODEL_NAME_OR_PATH,
+                filename=MODEL_FILE_PATTERN,
+                n_gpu_layers=N_GPU_LAYERS,
+                n_ctx=CONTEXT_SIZE,
+                n_threads=N_THREADS,
+                verbose=False
+            )
+        
+        logging.info("Model loaded successfully")
+        return llm
+    
+    except Exception as e:
+        logging.error(f"Error loading model: {e}")
+        raise
+
+
+ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "what how who where you").split()
+
+
+def main():
+    # Initialize model
+    model = get_model()
+    node = Node()
+
+    for event in node:
+        if event["type"] == "INPUT":
+            text = event["value"][0].as_py()
+            words = text.lower().split()
+
+            if any(word in ACTIVATION_WORDS for word in words):
+                # Generate response using system prompt
+                prompt = f"{SYSTEM_PROMPT}\nQ: {text}\nA:"
+                response = model(
+                    prompt,
+                    max_tokens=MAX_TOKENS,
+                    stop=["Q:", "\n"],
+                )["choices"][0]["text"]
+                
+                node.send_output(
+                    output_id="text", data=pa.array([response]), metadata={}
+                )
+
+
+if __name__ == "__main__":
+    main()

node-hub/dora-llama-cpp-python/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "dora-llama-cpp-python"
+version = "1.0.1"
+authors = [{ name = "Shashwat Patil", email = "email@email.com" }]
+description = "dora-llama-cpp-python"
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.9"
+
+dependencies = [
+    "dora-rs >= 0.3.9",
+    "torch == 2.4.0",
+    "torchvision >= 0.19",
+    "torchaudio >= 2.1.0",
+    "opencv-python >= 4.1.1",
+    "modelscope >= 1.18.1",
+    "mlx-lm>=0.21.1; sys_platform == 'darwin'",
+    "llama-cpp-python",
+]
+
+[tool.uv.sources]
+llama-cpp-python = [
+    { index = "llama_cpp_python_metal", marker = "sys_platform == 'darwin'" },
+    { index = "llama_cpp_python_cu121", marker = "sys_platform == 'linux'" },
+]
+
+[[tool.uv.index]]
+name = "llama_cpp_python_cu121"
+url = "https://abetlen.github.io/llama-cpp-python/whl/cu121"
+explicit = true
+
+[[tool.uv.index]]
+name = "llama_cpp_python_metal"
+url = "https://abetlen.github.io/llama-cpp-python/whl/metal"
+explicit = true
+
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
+
+[project.scripts]
+dora-llama-cpp-python = "dora_llama_cpp_python.main:main"

node-hub/dora-llama-cpp-python/test.yml

@@ -0,0 +1,68 @@
+nodes:
+  - id: dora-microphone
+    build: pip install -e ../../node-hub/dora-microphone
+    path: dora-microphone
+    inputs:
+      tick: dora/timer/millis/2000
+    outputs:
+      - audio
+
+  - id: dora-vad
+    build: pip install -e ../../node-hub/dora-vad
+    path: dora-vad
+    inputs:
+      audio: dora-microphone/audio
+    outputs:
+      - audio
+      - timestamp_start
+
+  - id: dora-distil-whisper
+    build: pip install -e ../../node-hub/dora-distil-whisper
+    path: dora-distil-whisper
+    inputs:
+      input: dora-vad/audio
+    outputs:
+      - text
+    env:
+      TARGET_LANGUAGE: english
+
+  - id: dora-llama-cpp-python
+    build: pip install -e ../../node-hub/dora-llama-cpp-python
+    path: dora-llama-cpp-python
+    inputs:
+      text: dora-distil-whisper/text
+    outputs:
+      - text
+    env:
+      MODEL_NAME_OR_PATH: TheBloke/Llama-2-7B-Chat-GGUF   # Llama 2.7B model pull from Hugging Face
+      MODEL_FILE_PATTERN: "*Q4_K_M.gguf"
+      SYSTEM_PROMPT: "You're a very succinct AI assistant with short answers."
+      ACTIVATION_WORDS: "what how who where you"
+      MAX_TOKENS: "512"
+      N_GPU_LAYERS: "35"     # Enable GPU acceleration
+      N_THREADS: "4"         # CPU threads
+      CONTEXT_SIZE: "4096"   # Maximum context window
+
+  - id: plot
+    build: pip install -e ../../node-hub/dora-rerun
+    path: dora-rerun
+    inputs:
+      text_llama: dora-llama-cpp-python/text  
+      text_whisper: dora-distil-whisper/text
+
+  - id: dora-kokoro-tts
+    build: pip install -e ../../node-hub/dora-kokoro-tts
+    path: dora-kokoro-tts
+    inputs:
+      text: dora-llama-cpp-python/text  
+    outputs:
+      - audio
+    env:
+      ACTIVATION_WORDS: you
+
+  - id: dora-pyaudio
+    build: pip install -e ../../node-hub/dora-pyaudio
+    path: dora-pyaudio
+    inputs:
+      audio: dora-kokoro-tts/audio
+      timestamp_start: dora-vad/timestamp_start

node-hub/dora-llama-cpp-python/tests/test_dora_llama_cpp_python.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+def test_import_main():
+    from dora_llama_cpp_python.main import main
+
+    # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow.
+    with pytest.raises(RuntimeError):
+        main()