From 93d25738b5902df4967d242041ad9d7e07e5ccfb Mon Sep 17 00:00:00 2001 From: krishna dubey Date: Sat, 15 Mar 2025 10:31:14 +0000 Subject: [PATCH 1/3] Add dora-lmdeploy node for LMDeploy-based inference --- node-hub/dora-lmdeploy/README.md | 22 ++++ .../dora-lmdeploy/dora_lmdeploy/__init__.py | 11 ++ .../dora_lmdeploy/chat_template.json | 3 + node-hub/dora-lmdeploy/dora_lmdeploy/main.py | 114 ++++++++++++++++++ node-hub/dora-lmdeploy/pyproject.toml | 36 ++++++ .../dora-lmdeploy/tests/test_dora_lmdeploy.py | 12 ++ 6 files changed, 198 insertions(+) create mode 100644 node-hub/dora-lmdeploy/README.md create mode 100644 node-hub/dora-lmdeploy/dora_lmdeploy/__init__.py create mode 100644 node-hub/dora-lmdeploy/dora_lmdeploy/chat_template.json create mode 100644 node-hub/dora-lmdeploy/dora_lmdeploy/main.py create mode 100644 node-hub/dora-lmdeploy/pyproject.toml create mode 100644 node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py diff --git a/node-hub/dora-lmdeploy/README.md b/node-hub/dora-lmdeploy/README.md new file mode 100644 index 00000000..787e5722 --- /dev/null +++ b/node-hub/dora-lmdeploy/README.md @@ -0,0 +1,22 @@ +# Dora LMDeploy Node + +Experimental Dora node for efficient inference using LMDeploy. + +## YAML Specification + +Use this node as follows: + +id: dora-lmdeploy +build: pip install dora-lmdeploy +path: dora-lmdeploy +inputs: +image: +source: camera/image +queue_size: 1 +text: dora-distil-whisper/text +outputs: + +text +env: +MODEL_NAME: "" +DEFAULT_PROMPT: "Describe the image briefly." \ No newline at end of file diff --git a/node-hub/dora-lmdeploy/dora_lmdeploy/__init__.py b/node-hub/dora-lmdeploy/dora_lmdeploy/__init__.py new file mode 100644 index 00000000..cde7a377 --- /dev/null +++ b/node-hub/dora-lmdeploy/dora_lmdeploy/__init__.py @@ -0,0 +1,11 @@ +import os + +# Define the path to the README file relative to the package directory +readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md") + +# Read the content of the README file +try: + with open(readme_path, encoding="utf-8") as f: + __doc__ = f.read() +except FileNotFoundError: + __doc__ = "README file not found." diff --git a/node-hub/dora-lmdeploy/dora_lmdeploy/chat_template.json b/node-hub/dora-lmdeploy/dora_lmdeploy/chat_template.json new file mode 100644 index 00000000..732bd68b --- /dev/null +++ b/node-hub/dora-lmdeploy/dora_lmdeploy/chat_template.json @@ -0,0 +1,3 @@ +{ + "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" +} \ No newline at end of file diff --git a/node-hub/dora-lmdeploy/dora_lmdeploy/main.py b/node-hub/dora-lmdeploy/dora_lmdeploy/main.py new file mode 100644 index 00000000..77f406d3 --- /dev/null +++ b/node-hub/dora-lmdeploy/dora_lmdeploy/main.py @@ -0,0 +1,114 @@ +"""Dora Node for LMDeploy-based inference.""" + +import os +import cv2 +import numpy as np +import pyarrow as pa +from dora import Node +from PIL import Image +from lmdeploy import pipeline + +# Environment variables +MODEL_NAME = os.getenv("MODEL_NAME", "internlm/internlm-xcomposer2-vl-7b") +DEFAULT_PROMPT = os.getenv("DEFAULT_PROMPT", "Describe this image.") +IMAGE_RESIZE_RATIO = float(os.getenv("IMAGE_RESIZE_RATIO", "1.0")) +ACTIVATION_WORDS = os.getenv("ACTIVATION_WORDS", "").split() + +# Initialize LMDeploy pipeline +pipe = pipeline(model_name=MODEL_NAME, backend_config={"backend": "tensorrt"}) + + +def generate(frames: dict, question: str, image_id=None): + """Generate a response using LMDeploy given images and a question.""" + if image_id is not None: + images = [frames[image_id]] + else: + images = list(frames.values()) + + # Perform inference using LMDeploy pipeline + result = pipe((question, images)) + return result.text.strip() + + +def main(): + """Main function to handle events in the Dora node.""" + pa.array([]) # Initialize PyArrow array + node = Node() + + frames = {} + cached_text = DEFAULT_PROMPT + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if "image" in event_id: + # Process incoming image data + storage = event["value"] + metadata = event["metadata"] + encoding = metadata["encoding"] + width = metadata["width"] + height = metadata["height"] + + if encoding in ["bgr8", "rgb8", "jpeg", "jpg", "jpe", "bmp", "webp", "png"]: + channels = 3 + storage_type = np.uint8 + else: + raise RuntimeError(f"Unsupported image encoding: {encoding}") + + if encoding == "bgr8": + frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) + ) + frame = frame[:, :, ::-1] # Convert BGR to RGB + elif encoding == "rgb8": + frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) + ) + elif encoding in ["jpeg", "jpg", "jpe", "bmp", "webp", "png"]: + storage = storage.to_numpy() + frame = cv2.imdecode(storage, cv2.IMREAD_COLOR) + frame = frame[:, :, ::-1] # Convert BGR to RGB + else: + raise RuntimeError(f"Unsupported image encoding: {encoding}") + + image = Image.fromarray(frame) + frames[event_id] = image + + elif "text" in event_id: + # Process incoming text data + if len(event["value"]) > 0: + text = event["value"][0].as_py() + image_id = event["metadata"].get("image_id", None) + else: + text = cached_text + + words = text.split() + if len(ACTIVATION_WORDS) > 0 and all(word not in ACTIVATION_WORDS for word in words): + continue + + cached_text = text + + if len(frames.keys()) == 0: + continue + + # Generate response using LMDeploy pipeline + response = generate(frames, text, image_id) + node.send_output( + "text", + pa.array([response]), + {"image_id": image_id if image_id is not None else "all"}, + ) + + elif event_type == "ERROR": + print("Event Error:" + event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/dora-lmdeploy/pyproject.toml b/node-hub/dora-lmdeploy/pyproject.toml new file mode 100644 index 00000000..759173ec --- /dev/null +++ b/node-hub/dora-lmdeploy/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "dora-lmdeploy" +version = "0.1.0" +authors = [ + { name = "Haixuan Xavier Tao", email = "tao.xavier@outlook.com" }, + { name = "Enzo Le Van", email = "dev@enzo-le-van.fr" }, +] +description = "Dora Node for inference using LMDeploy" +license = { text = "MIT" } +readme = "README.md" +requires-python = ">=3.9" + +dependencies = [ + "dora-rs >= 0.3.9", + "numpy < 2.0.0", + "torch == 2.4.0", + "torchvision >= 0.19", + "torchaudio >= 2.1.0", + "lmdeploy >= 0.2.0", + "opencv-python >= 4.1.1", +] + +[dependency-groups] +dev = ["pytest >=8.1.1", "ruff >=0.9.1"] + +[project.scripts] +dora-lmdeploy = "dora_lmdeploy.main:main" + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.ruff.lint] +extend-select = [ + "D", # pydocstyle +] diff --git a/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py new file mode 100644 index 00000000..6127a14c --- /dev/null +++ b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py @@ -0,0 +1,12 @@ +"""Tests for the dora-lmdeploy node.""" + +import pytest + + +def test_import_main(): + """Test that the main function in dora-lmdeploy can be imported and executed.""" + from dora_lmdeploy.main import main + + # Check that everything is working, and catch Dora RuntimeException as we're not running in a Dora dataflow. + with pytest.raises(RuntimeError): + main() From ad98015ba1de52924da73c563a53f7e17fe6e529 Mon Sep 17 00:00:00 2001 From: krishna dubey Date: Sun, 16 Mar 2025 03:35:39 +0000 Subject: [PATCH 2/3] fixed error due to Node class --- node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py index 6127a14c..5b263a34 100644 --- a/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py +++ b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py @@ -1,12 +1,9 @@ -"""Tests for the dora-lmdeploy node.""" - import pytest - def test_import_main(): """Test that the main function in dora-lmdeploy can be imported and executed.""" from dora_lmdeploy.main import main - # Check that everything is working, and catch Dora RuntimeException as we're not running in a Dora dataflow. - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match=".*Node.*"): main() + From 053a202d448c331bc4eee9f6e0896880f93fb0d4 Mon Sep 17 00:00:00 2001 From: krishna dubey Date: Sun, 16 Mar 2025 03:54:11 +0000 Subject: [PATCH 3/3] Fix D205: Add blank line between summary and description in module-level docstring --- node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py index 5b263a34..53ed7632 100644 --- a/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py +++ b/node-hub/dora-lmdeploy/tests/test_dora_lmdeploy.py @@ -1,7 +1,17 @@ +"""Module for testing Dora LMDeploy functionality. + +This module contains tests for verifying the functionality of the Dora LMDeploy component. +""" + + import pytest def test_import_main(): - """Test that the main function in dora-lmdeploy can be imported and executed.""" + """Test that the main function in Dora LMDeploy can be imported and executed. + + This test verifies that the `main` function can be imported and runs without errors, + catching any expected RuntimeError when executed outside a valid Dora dataflow context. + """ from dora_lmdeploy.main import main with pytest.raises(RuntimeError, match=".*Node.*"):