Added dora-phi4 inside node-hub (#861)

#814 Issue fix

* Added phi4 multimodel

node-hub/dora-phi4/README.md

@@ -0,0 +1,40 @@
+# dora-phi4
+
+## Getting started
+
+- Install it with uv:
+
+```bash
+uv venv -p 3.11 --seed
+uv pip install -e .
+```
+
+## Contribution Guide
+
+- Format with [ruff](https://docs.astral.sh/ruff/):
+
+```bash
+uv pip install ruff
+uv run ruff check . --fix
+```
+
+- Lint with ruff:
+
+```bash
+uv run ruff check .
+```
+
+- Test with [pytest](https://github.com/pytest-dev/pytest)
+
+```bash
+uv pip install pytest
+uv run pytest . # Test
+```
+
+## YAML Specification
+
+## Examples
+
+## License
+
+dora-phi4's code are released under the MIT License

node-hub/dora-phi4/dora_phi4/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+    with open(readme_path, encoding="utf-8") as f:
+        __doc__ = f.read()
+except FileNotFoundError:
+    __doc__ = "README file not found."

node-hub/dora-phi4/dora_phi4/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

node-hub/dora-phi4/dora_phi4/main.py

@@ -0,0 +1,91 @@
+import pyarrow as pa
+import torch
+from accelerate import infer_auto_device_map
+from dora import Node
+from transformers import (
+    AutoModelForCausalLM,
+    AutoProcessor,
+    GenerationConfig,
+)
+
+# 🔍 Detect the best available device
+if torch.cuda.is_available():
+    device = "cuda"
+    torch_dtype = torch.float16  # Use float16 for efficiency
+# TODO: Uncomment this once phi4 support mps backend.
+# elif torch.backends.mps.is_available():
+# device = "mps"
+# torch_dtype = torch.float16  # Reduce memory usage for MPS
+else:
+    device = "cpu"
+    torch_dtype = torch.bfloat16  # CPU uses bfloat16 for efficiency
+
+
+# Load the model and processor
+MODEL_PATH = "microsoft/Phi-4-multimodal-instruct"
+
+processor = AutoProcessor.from_pretrained(
+    MODEL_PATH, trust_remote_code=True, use_fast=True
+)
+
+# Define model config
+MODEL_CONFIG = {
+    "torch_dtype": torch_dtype,
+    "trust_remote_code": True,
+    "_attn_implementation": "flash_attention_2"
+    if device == "cuda" and torch.cuda.get_device_properties(0).total_memory > 16e9
+    else "eager",
+    "low_cpu_mem_usage": True,
+}
+
+# Infer device map without full initialization
+device_map = infer_auto_device_map(
+    AutoModelForCausalLM.from_pretrained(MODEL_PATH, **MODEL_CONFIG)
+)
+
+# Load the model directly with the inferred device map
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_PATH, **MODEL_CONFIG, device_map=device_map
+)
+
+generation_config = GenerationConfig.from_pretrained(MODEL_PATH)
+
+user_prompt = "<|user|>"
+assistant_prompt = "<|assistant|>"
+prompt_suffix = "<|end|>"
+
+
+def main():
+    node = Node()
+
+    for event in node:
+        if event["type"] == "INPUT":
+            input_id = event["id"]
+            if input_id == "text":
+                text = event["value"][0].as_py()
+                prompt = f"{user_prompt}{text}{prompt_suffix}{assistant_prompt}"
+
+                # Process input
+                inputs = processor(
+                    text=prompt,
+                    return_tensors="pt",
+                ).to(model.device)
+                # Generate response
+                with torch.no_grad():
+                    generate_ids = model.generate(
+                        **inputs,
+                        max_new_tokens=512,
+                        generation_config=generation_config,
+                    )
+                    generate_ids = generate_ids[:, inputs["input_ids"].shape[1] :]
+
+                response = processor.batch_decode(
+                    generate_ids,
+                    skip_special_tokens=True,
+                    clean_up_tokenization_spaces=False,
+                )[0]
+                node.send_output("text", pa.array([response]))
+
+
+if __name__ == "__main__":
+    main()

node-hub/dora-phi4/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "dora-phi4"
+version = "0.0.0"
+authors = [{ name = "Somay", email = "ssomay2002@gmail.com" }]
+description = "DORA node for Phi-4 multimodal model"
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.10"
+
+dependencies = [
+    "dora-rs>=0.3.9",
+    "torch==2.6.0",
+    "torchvision==0.21.0",
+    "transformers==4.48.2",
+    "accelerate==1.3.0",
+    "soundfile==0.13.1",
+    "pillow==11.1.0",
+    "scipy==1.15.2",
+    "backoff==2.2.1",
+    "peft==0.13.2",
+    "bitsandbytes>=0.42.0",
+    "requests"
+]
+
+[tool.setuptools]
+packages = ["dora_phi4"]
+
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
+
+[project.scripts]
+dora-phi4 = "dora_phi4.main:main"

node-hub/dora-phi4/tests/test_dora_phi4.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+def test_import_main():
+    from dora_phi4.main import main
+
+    # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow.
+    with pytest.raises(RuntimeError):
+        main()

node-hub/pyarrow-assert/pyarrow_assert/main.py

@@ -7,8 +7,6 @@ import os
 import pyarrow as pa
 from dora import Node
 
-RUNNER_CI = True if os.getenv("CI") == "true" else False
-
 
 def main():
     # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables.
@@ -38,11 +36,12 @@ def main():
         args.name,
     )  # provide the name to connect to the dataflow if dynamic node
 
-    data = ast.literal_eval(data)
+    try:
+        data = ast.literal_eval(data)
+    except Exception:  # noqa
+        print("Passing input as string")
 
-    if isinstance(data, list):
-        data = pa.array(data)  # initialize pyarrow array
-    elif isinstance(data, str) or isinstance(data, int) or isinstance(data, float):
+    if isinstance(data, (str, int, float)):
         data = pa.array([data])
     else:
         data = pa.array(data)  # initialize pyarrow array

node-hub/pyarrow-sender/pyarrow_sender/main.py

@@ -7,8 +7,6 @@ import os
 import pyarrow as pa
 from dora import Node
 
-RUNNER_CI = True if os.getenv("CI") == "true" else False
-
 
 def main():
     # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables.
@@ -44,11 +42,10 @@ def main():
         )
     try:
         data = ast.literal_eval(data)
-    except ValueError:
+    except Exception:  # noqa
         print("Passing input as string")
-    if isinstance(data, list):
-        data = pa.array(data)  # initialize pyarrow array
-    elif isinstance(data, str) or isinstance(data, int) or isinstance(data, float):
+
+    if isinstance(data, (str, int, float)):
         data = pa.array([data])
     else:
         data = pa.array(data)  # initialize pyarrow array

tests/llm/README.md

@@ -0,0 +1,10 @@
+# Test an LLM on a simple prompt
+
+To run:
+
+```bash
+cd tests/llm
+uv venv --seed -p 3.11
+dora build phi4.yaml --uv
+dora run phi4.yaml --uv
+```

tests/llm/phi4.yaml

@@ -0,0 +1,24 @@
+nodes:
+  - id: pyarrow-sender
+    build: pip install -e ../../node-hub/pyarrow-sender
+    path: pyarrow-sender
+    outputs:
+      - data
+    env:
+      DATA: "'Please only generate the following output: This is a test'"
+
+  - id: dora-phi4
+    build: pip install -e ../../node-hub/dora-phi4
+    path: dora-phi4
+    inputs:
+      text: pyarrow-sender/data
+    outputs:
+      - text
+
+  - id: pyarrow-assert
+    build: pip install -e ../../node-hub/pyarrow-assert
+    path: pyarrow-assert
+    inputs:
+      data: dora-phi4/text
+    env:
+      DATA: "'This is a test'"

tests/llm/qwen2.5.yaml

@@ -0,0 +1,24 @@
+nodes:
+  - id: pyarrow-sender
+    build: pip install -e ../../node-hub/pyarrow-sender
+    path: pyarrow-sender
+    outputs:
+      - data
+    env:
+      DATA: "'Please only output: This is a test'"
+
+  - id: dora-qwen2.5
+    build: pip install -e ../../node-hub/dora-qwen2.5
+    path: dora-qwen2-5
+    inputs:
+      text: pyarrow-sender/data
+    outputs:
+      - text
+
+  - id: pyarrow-assert
+    build: pip install -e ../../node-hub/pyarrow-assert
+    path: pyarrow-assert
+    inputs:
+      data: dora-phi4/text
+    env:
+      DATA: "'This is a test'"