From 92eaed1c911684e36639c00655568f5c8639ae56 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Jun 2024 19:31:48 +0200 Subject: [PATCH 01/18] Move `rerun` and `record` into nodes_hub --- {tool_nodes => nodes_hub}/dora-record/Cargo.toml | 0 {tool_nodes => nodes_hub}/dora-record/README.md | 0 {tool_nodes => nodes_hub}/dora-record/src/main.rs | 0 {tool_nodes => nodes_hub}/dora-rerun/Cargo.toml | 0 {tool_nodes => nodes_hub}/dora-rerun/README.md | 0 {tool_nodes => nodes_hub}/dora-rerun/src/main.rs | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {tool_nodes => nodes_hub}/dora-record/Cargo.toml (100%) rename {tool_nodes => nodes_hub}/dora-record/README.md (100%) rename {tool_nodes => nodes_hub}/dora-record/src/main.rs (100%) rename {tool_nodes => nodes_hub}/dora-rerun/Cargo.toml (100%) rename {tool_nodes => nodes_hub}/dora-rerun/README.md (100%) rename {tool_nodes => nodes_hub}/dora-rerun/src/main.rs (100%) diff --git a/tool_nodes/dora-record/Cargo.toml b/nodes_hub/dora-record/Cargo.toml similarity index 100% rename from tool_nodes/dora-record/Cargo.toml rename to nodes_hub/dora-record/Cargo.toml diff --git a/tool_nodes/dora-record/README.md b/nodes_hub/dora-record/README.md similarity index 100% rename from tool_nodes/dora-record/README.md rename to nodes_hub/dora-record/README.md diff --git a/tool_nodes/dora-record/src/main.rs b/nodes_hub/dora-record/src/main.rs similarity index 100% rename from tool_nodes/dora-record/src/main.rs rename to nodes_hub/dora-record/src/main.rs diff --git a/tool_nodes/dora-rerun/Cargo.toml b/nodes_hub/dora-rerun/Cargo.toml similarity index 100% rename from tool_nodes/dora-rerun/Cargo.toml rename to nodes_hub/dora-rerun/Cargo.toml diff --git a/tool_nodes/dora-rerun/README.md b/nodes_hub/dora-rerun/README.md similarity index 100% rename from tool_nodes/dora-rerun/README.md rename to nodes_hub/dora-rerun/README.md diff --git a/tool_nodes/dora-rerun/src/main.rs b/nodes_hub/dora-rerun/src/main.rs similarity index 100% rename from tool_nodes/dora-rerun/src/main.rs rename to nodes_hub/dora-rerun/src/main.rs From c0ba3f221a61fb6322f68b59a5ca259acd995535 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Jun 2024 19:32:27 +0200 Subject: [PATCH 02/18] Adding python node hub --- nodes_hub/opencv-plot/plot.py | 73 +++++++++++++++++++ nodes_hub/opencv-plot/requirements.txt | 2 + .../opencv-video-capture/requirements.txt | 3 + .../opencv-video-capture/video_capture.py | 52 +++++++++++++ nodes_hub/ultralytics-yolo/requirements.txt | 3 + nodes_hub/ultralytics-yolo/yolo.py | 50 +++++++++++++ 6 files changed, 183 insertions(+) create mode 100644 nodes_hub/opencv-plot/plot.py create mode 100644 nodes_hub/opencv-plot/requirements.txt create mode 100644 nodes_hub/opencv-video-capture/requirements.txt create mode 100644 nodes_hub/opencv-video-capture/video_capture.py create mode 100644 nodes_hub/ultralytics-yolo/requirements.txt create mode 100644 nodes_hub/ultralytics-yolo/yolo.py diff --git a/nodes_hub/opencv-plot/plot.py b/nodes_hub/opencv-plot/plot.py new file mode 100644 index 00000000..d2c4ce74 --- /dev/null +++ b/nodes_hub/opencv-plot/plot.py @@ -0,0 +1,73 @@ +import os +from dataclasses import dataclass + +import cv2 +import numpy as np + +from dora import Node + +CI = os.environ.get("CI") + +IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) +IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) + +FONT = cv2.FONT_HERSHEY_SIMPLEX + + +@dataclass +class Plotter: + frame: np.array = np.array([]) + bboxes: np.array = np.array([[]]) + conf: np.array = np.array([]) + label: np.array = np.array([]) + + +if __name__ == "__main__": + plotter = Plotter() + node = Node() + for event in node: + event_type = event["type"] + if event_type == "INPUT": + if event["id"] == "image": + frame = event["value"].to_numpy() + frame = frame.reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)).copy() + plotter.frame = frame + + elif event["id"] == "bbox": + bboxes = event["value"][0]["bbox"].values.to_numpy() + conf = event["value"][0]["conf"].values.to_numpy() + label = event["value"][0]["names"].values.to_pylist() + plotter.bboxes = np.reshape(bboxes, (-1, 4)) + plotter.conf = conf + plotter.label = label + continue + + for bbox in zip(plotter.bboxes, plotter.conf, plotter.label): + [ + [min_x, min_y, max_x, max_y], + confidence, + label, + ] = bbox + cv2.rectangle( + plotter.frame, + (int(min_x), int(min_y)), + (int(max_x), int(max_y)), + (0, 255, 0), + 2, + ) + + cv2.putText( + plotter.frame, + f"{label}, {confidence:0.2f}", + (int(max_x) - 120, int(max_y) - 10), + FONT, + 0.5, + (0, 255, 0), + 2, + 1, + ) + + if CI != "true": + cv2.imshow("frame", plotter.frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break diff --git a/nodes_hub/opencv-plot/requirements.txt b/nodes_hub/opencv-plot/requirements.txt new file mode 100644 index 00000000..7f3ef381 --- /dev/null +++ b/nodes_hub/opencv-plot/requirements.txt @@ -0,0 +1,2 @@ +numpy<2.0.0 +opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/requirements.txt b/nodes_hub/opencv-video-capture/requirements.txt new file mode 100644 index 00000000..1db957a5 --- /dev/null +++ b/nodes_hub/opencv-video-capture/requirements.txt @@ -0,0 +1,3 @@ +numpy<2.0.0 +pyarrow +opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/video_capture.py b/nodes_hub/opencv-video-capture/video_capture.py new file mode 100644 index 00000000..cad4320c --- /dev/null +++ b/nodes_hub/opencv-video-capture/video_capture.py @@ -0,0 +1,52 @@ +import os +import time + +import cv2 +import numpy as np +import pyarrow as pa + +from dora import Node + +CAM_INDEX = int(os.getenv("CAM_INDEX", "0")) +IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) +IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) +MAX_DURATION = int(os.getenv("DURATION", "20")) +FONT = cv2.FONT_HERSHEY_SIMPLEX + + +start = time.time() + + +if __name__ == "__main__": + + video_capture = cv2.VideoCapture(CAM_INDEX) + node = Node() + + while time.time() - start < MAX_DURATION: + event = node.next() + if event is None: + break + if event is not None: + event_type = event["type"] + if event_type == "INPUT": + ret, frame = video_capture.read() + + # Fail to read camera + if not ret: + frame = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3), dtype=np.uint8) + cv2.putText( + frame, + "No Webcam was found at index %d" % (CAM_INDEX), + (int(30), int(30)), + FONT, + 0.75, + (255, 255, 255), + 2, + 1, + ) + + node.send_output( + "image", + pa.array(frame.ravel()), + event["metadata"], + ) diff --git a/nodes_hub/ultralytics-yolo/requirements.txt b/nodes_hub/ultralytics-yolo/requirements.txt new file mode 100644 index 00000000..d8872ab6 --- /dev/null +++ b/nodes_hub/ultralytics-yolo/requirements.txt @@ -0,0 +1,3 @@ +numpy<2.0.0 +pyarrow +ultralytics \ No newline at end of file diff --git a/nodes_hub/ultralytics-yolo/yolo.py b/nodes_hub/ultralytics-yolo/yolo.py new file mode 100644 index 00000000..392c3ee9 --- /dev/null +++ b/nodes_hub/ultralytics-yolo/yolo.py @@ -0,0 +1,50 @@ +## Imports +import os + +import numpy as np +import pyarrow as pa +from ultralytics import YOLO + +from dora import Node + +## OS Environment variable +IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) +IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) +MODEL = os.getenv("MODEL", "yolov8n.pt") + +if __name__ == "__main__": + + model = YOLO(MODEL) + + node = Node("object_detection") + + for event in node: + event_type = event["type"] + if event_type == "INPUT": + event_id = event["id"] + if event_id == "image": + frame = ( + event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) + ) + frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + results = model(frame, verbose=False) # includes NMS + # Process results + bboxes = np.array(results[0].boxes.xyxy.cpu()) + conf = np.array(results[0].boxes.conf.cpu()) + labels = np.array(results[0].boxes.cls.cpu()) + names = [model.names.get(label) for label in labels] + + node.send_output( + "bbox", + pa.array( + [ + { + "bbox": bboxes.ravel(), + "conf": conf, + "labels": labels, + "names": names, + } + ] + ), + event["metadata"], + ) From b0df85c6ec09d513793c1ce8e4c423b618aaab48 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 27 Jun 2024 19:32:50 +0200 Subject: [PATCH 03/18] simplifying examples to use node hub --- Cargo.toml | 4 +- examples/python-dataflow/dataflow.yml | 38 ++--- examples/python-dataflow/dataflow_dynamic.yml | 23 ++- examples/python-dataflow/example.py | 5 - examples/python-dataflow/object_detection.py | 40 ----- examples/python-dataflow/plot.py | 96 ------------ examples/python-dataflow/plot_dynamic.py | 148 ++++++++---------- examples/python-dataflow/requirements.txt | 47 ------ examples/python-dataflow/run.rs | 15 -- examples/python-dataflow/utils.py | 82 ---------- examples/python-dataflow/webcam.py | 52 ------ 11 files changed, 93 insertions(+), 457 deletions(-) delete mode 100644 examples/python-dataflow/example.py delete mode 100755 examples/python-dataflow/object_detection.py delete mode 100755 examples/python-dataflow/plot.py mode change 100755 => 100644 examples/python-dataflow/plot_dynamic.py delete mode 100644 examples/python-dataflow/requirements.txt delete mode 100644 examples/python-dataflow/utils.py delete mode 100755 examples/python-dataflow/webcam.py diff --git a/Cargo.toml b/Cargo.toml index 798b7a2a..fa52a3b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,8 @@ members = [ "libraries/shared-memory-server", "libraries/extensions/download", "libraries/extensions/telemetry/*", - "tool_nodes/dora-record", - "tool_nodes/dora-rerun", + "nodes_hub/dora-record", + "nodes_hub/dora-rerun", "libraries/extensions/ros2-bridge", "libraries/extensions/ros2-bridge/msg-gen", "libraries/extensions/ros2-bridge/python", diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 612ce410..2b608ce7 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -1,25 +1,25 @@ nodes: - id: webcam - custom: - source: ./webcam.py - inputs: - tick: - source: dora/timer/millis/50 - queue_size: 1000 - outputs: - - image + build: pip install -r ../../nodes_hub/opencv-video-capture/requirements.txt + path: ../../nodes_hub/opencv-video-capture/video_capture.py + inputs: + tick: dora/timer/millis/50 + outputs: + - image + env: + DURATION: 100 - id: object_detection - custom: - source: ./object_detection.py - inputs: - image: webcam/image - outputs: - - bbox + build: pip install -r ../../nodes_hub/ultralytics-yolo/requirements.txt + path: ../../nodes_hub/ultralytics-yolo/yolo.py + inputs: + image: webcam/image + outputs: + - bbox - id: plot - custom: - source: ./plot.py - inputs: - image: webcam/image - bbox: object_detection/bbox + build: pip install -r ../../nodes_hub/opencv-plot/requirements.txt + path: ../../nodes_hub/opencv-plot/plot.py + inputs: + image: webcam/image + bbox: object_detection/bbox diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 677f8a7f..3c9ae1cb 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -1,16 +1,15 @@ nodes: - id: webcam - custom: - source: ./webcam.py - inputs: - tick: - source: dora/timer/millis/50 - queue_size: 1000 - outputs: - - image + build: pip install -r ../../nodes_hub/opencv-webcam/requirements.txt + path: ../../nodes_hub/opencv-webcam/webcam.py + inputs: + tick: + source: dora/timer/millis/50 + queue_size: 1000 + outputs: + - image - id: plot - custom: - source: dynamic - inputs: - image: webcam/image + path: dynamic + inputs: + image: webcam/image diff --git a/examples/python-dataflow/example.py b/examples/python-dataflow/example.py deleted file mode 100644 index c9221a3a..00000000 --- a/examples/python-dataflow/example.py +++ /dev/null @@ -1,5 +0,0 @@ -from dora import Node - -node = Node("plot") - -event = node.next() diff --git a/examples/python-dataflow/object_detection.py b/examples/python-dataflow/object_detection.py deleted file mode 100755 index 70a0e712..00000000 --- a/examples/python-dataflow/object_detection.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import cv2 -import numpy as np -from ultralytics import YOLO - -from dora import Node -import pyarrow as pa - -model = YOLO("yolov8n.pt") - -node = Node() - -for event in node: - event_type = event["type"] - if event_type == "INPUT": - event_id = event["id"] - if event_id == "image": - print("[object detection] received image input") - frame = event["value"].to_numpy() - frame = cv2.imdecode(frame, -1) - frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) - results = model(frame) # includes NMS - # Process results - boxes = np.array(results[0].boxes.xyxy.cpu()) - conf = np.array(results[0].boxes.conf.cpu()) - label = np.array(results[0].boxes.cls.cpu()) - # concatenate them together - arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) - - node.send_output("bbox", pa.array(arrays.ravel()), event["metadata"]) - else: - print("[object detection] ignoring unexpected input:", event_id) - elif event_type == "STOP": - print("[object detection] received stop") - elif event_type == "ERROR": - print("[object detection] error: ", event["error"]) - else: - print("[object detection] received unexpected event:", event_type) diff --git a/examples/python-dataflow/plot.py b/examples/python-dataflow/plot.py deleted file mode 100755 index 035fc41d..00000000 --- a/examples/python-dataflow/plot.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -from dora import Node -from dora import DoraStatus - -import cv2 -import numpy as np -from utils import LABELS - -CI = os.environ.get("CI") - -font = cv2.FONT_HERSHEY_SIMPLEX - - -class Plotter: - """ - Plot image and bounding box - """ - - def __init__(self): - self.image = [] - self.bboxs = [] - - def on_input( - self, - dora_input, - ) -> DoraStatus: - """ - Put image and bounding box on cv2 window. - - Args: - dora_input["id"] (str): Id of the dora_input declared in the yaml configuration - dora_input["value"] (arrow array): message of the dora_input - """ - if dora_input["id"] == "image": - frame = dora_input["value"].to_numpy() - frame = cv2.imdecode(frame, -1) - self.image = frame - - elif dora_input["id"] == "bbox" and len(self.image) != 0: - bboxs = dora_input["value"].to_numpy() - self.bboxs = np.reshape(bboxs, (-1, 6)) - for bbox in self.bboxs: - [ - min_x, - min_y, - max_x, - max_y, - confidence, - label, - ] = bbox - cv2.rectangle( - self.image, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - self.image, - LABELS[int(label)] + f", {confidence:0.2f}", - (int(max_x), int(max_y)), - font, - 0.75, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", self.image) - if cv2.waitKey(1) & 0xFF == ord("q"): - return DoraStatus.STOP - - return DoraStatus.CONTINUE - - -plotter = Plotter() -node = Node() - -for event in node: - event_type = event["type"] - if event_type == "INPUT": - status = plotter.on_input(event) - if status == DoraStatus.CONTINUE: - pass - elif status == DoraStatus.STOP: - print("plotter returned stop status") - break - elif event_type == "STOP": - print("received stop") - else: - print("received unexpected event:", event_type) diff --git a/examples/python-dataflow/plot_dynamic.py b/examples/python-dataflow/plot_dynamic.py old mode 100755 new mode 100644 index b3eda8b7..31c76ee2 --- a/examples/python-dataflow/plot_dynamic.py +++ b/examples/python-dataflow/plot_dynamic.py @@ -1,97 +1,71 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - import os -from dora import Node -from dora import DoraStatus +from dataclasses import dataclass import cv2 import numpy as np -from utils import LABELS - -CI = os.environ.get("CI") - -font = cv2.FONT_HERSHEY_SIMPLEX - - -class Plotter: - """ - Plot image and bounding box - """ - def __init__(self): - self.image = [] - self.bboxs = [] - - def on_input( - self, - dora_input, - ) -> DoraStatus: - """ - Put image and bounding box on cv2 window. - - Args: - dora_input["id"] (str): Id of the dora_input declared in the yaml configuration - dora_input["value"] (arrow array): message of the dora_input - """ - if dora_input["id"] == "image": - frame = dora_input["value"].to_numpy() - frame = cv2.imdecode(frame, -1) - self.image = frame - - elif dora_input["id"] == "bbox" and len(self.image) != 0: - bboxs = dora_input["value"].to_numpy() - self.bboxs = np.reshape(bboxs, (-1, 6)) - for bbox in self.bboxs: - [ - min_x, - min_y, - max_x, - max_y, - confidence, - label, - ] = bbox - cv2.rectangle( - self.image, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - self.image, - LABELS[int(label)] + f", {confidence:0.2f}", - (int(max_x), int(max_y)), - font, - 0.75, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", self.image) - if cv2.waitKey(1) & 0xFF == ord("q"): - return DoraStatus.STOP +from dora import Node - return DoraStatus.CONTINUE +CI = os.environ.get("CI") +IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) +IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) -plotter = Plotter() +FONT = cv2.FONT_HERSHEY_SIMPLEX -node = Node("plot") -for event in node: - event_type = event["type"] - if event_type == "INPUT": - status = plotter.on_input(event) - if status == DoraStatus.CONTINUE: - pass - elif status == DoraStatus.STOP: - print("plotter returned stop status") - break - elif event_type == "STOP": - print("received stop") - else: - print("received unexpected event:", event_type) +@dataclass +class Plotter: + frame: np.array = np.array([]) + bboxes: np.array = np.array([]) + + +if __name__ == "__main__": + plotter = Plotter() + node = Node("plot") + + for event in node: + event_type = event["type"] + if event_type == "INPUT": + if event["id"] == "image": + frame = event["value"].to_numpy() + frame = ( + event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) + ) + plotter.frame = frame + + elif event["id"] == "bbox" and len(plotter.frame) != 0: + bboxs = event["value"].to_numpy() + plotter.bboxes = np.reshape(bboxs, (-1, 6)) + for bbox in plotter.bboxs: + [ + min_x, + min_y, + max_x, + max_y, + confidence, + label, + ] = bbox + cv2.rectangle( + plotter.frame, + (int(min_x), int(min_y)), + (int(max_x), int(max_y)), + (0, 255, 0), + 2, + ) + + cv2.putText( + plotter.frame, + LABELS[int(label)] + f", {confidence:0.2f}", + (int(max_x), int(max_y)), + FONT, + 0.75, + (0, 255, 0), + 2, + 1, + ) + + if CI != "true": + cv2.imshow("frame", plotter.frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break diff --git a/examples/python-dataflow/requirements.txt b/examples/python-dataflow/requirements.txt deleted file mode 100644 index 9c1fc915..00000000 --- a/examples/python-dataflow/requirements.txt +++ /dev/null @@ -1,47 +0,0 @@ -# YOLOv5 requirements -# Usage: pip install -r requirements.txt - -# Base ---------------------------------------- -ultralytics -gitpython -ipython # interactive notebook -matplotlib>=3.2.2 -numpy<2.0.0 # See: https://github.com/opencv/opencv-python/issues/997 -opencv-python>=4.1.1 -Pillow>=7.1.2 -psutil # system resources -PyYAML>=5.3.1 -requests>=2.23.0 -scipy>=1.4.1 -thop>=0.1.1 # FLOPs computation -torch # see https://pytorch.org/get-started/locally (recommended) -torchvision -tqdm>=4.64.0 - -# Logging ------------------------------------- -tensorboard>=2.4.1 -# wandb -# clearml - -# Plotting ------------------------------------ -pandas>=1.1.4 -seaborn>=0.11.0 - -# Export -------------------------------------- -# coremltools>=5.2 # CoreML export -# onnx>=1.9.0 # ONNX export -# onnx-simplifier>=0.4.1 # ONNX simplifier -# nvidia-pyindex # TensorRT export -# nvidia-tensorrt # TensorRT export -# scikit-learn==0.19.2 # CoreML quantization -# tensorflow>=2.4.1 # TFLite export (or tensorflow-cpu, tensorflow-aarch64) -# tensorflowjs>=3.9.0 # TF.js export -# openvino-dev # OpenVINO export - -# Extras -------------------------------------- -# albumentations>=1.0.3 -# pycocotools>=2.0 # COCO mAP -# roboflow - -opencv-python>=4.1.1 -maturin diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 65ae5831..20fdc399 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -50,21 +50,6 @@ async fn main() -> eyre::Result<()> { ); } - run( - get_python_path().context("Could not get pip binary")?, - &["-m", "pip", "install", "--upgrade", "pip"], - None, - ) - .await - .context("failed to install pip")?; - run( - get_pip_path().context("Could not get pip binary")?, - &["install", "-r", "requirements.txt"], - None, - ) - .await - .context("pip install failed")?; - run( "maturin", &["develop"], diff --git a/examples/python-dataflow/utils.py b/examples/python-dataflow/utils.py deleted file mode 100644 index dabc915e..00000000 --- a/examples/python-dataflow/utils.py +++ /dev/null @@ -1,82 +0,0 @@ -LABELS = [ - "ABC", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "backpack", - "umbrella", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "dining table", - "toilet", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush", -] diff --git a/examples/python-dataflow/webcam.py b/examples/python-dataflow/webcam.py deleted file mode 100755 index 00b47f27..00000000 --- a/examples/python-dataflow/webcam.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import time -import numpy as np -import cv2 - -from dora import Node - -node = Node() - -CAMERA_INDEX = int(os.getenv("CAMERA_INDEX", 0)) -CAMERA_WIDTH = 640 -CAMERA_HEIGHT = 480 -video_capture = cv2.VideoCapture(CAMERA_INDEX) -font = cv2.FONT_HERSHEY_SIMPLEX - -start = time.time() - -# Run for 20 seconds -while time.time() - start < 10: - # Wait next dora_input - event = node.next() - event_type = event["type"] - if event_type == "INPUT": - ret, frame = video_capture.read() - if not ret: - frame = np.zeros((CAMERA_HEIGHT, CAMERA_WIDTH, 3), dtype=np.uint8) - cv2.putText( - frame, - "No Webcam was found at index %d" % (CAMERA_INDEX), - (int(30), int(30)), - font, - 0.75, - (255, 255, 255), - 2, - 1, - ) - node.send_output( - "image", - cv2.imencode(".jpg", frame)[1].tobytes(), - event["metadata"], - ) - elif event_type == "STOP": - print("received stop") - break - else: - print("received unexpected event:", event_type) - break - -video_capture.release() From 05caf8fbfdcea058677619ca01e094af922fc108 Mon Sep 17 00:00:00 2001 From: Hennzau Date: Tue, 9 Jul 2024 19:23:55 +0200 Subject: [PATCH 04/18] nodes_hub -> node-hub + opencv-plot and opencv-video-capture .idea .idea README.md and better outputs/inputs README.md and ultralytics-yolo fix queue_size and dynamic README.md Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI --- .github/workflows/ci.yml | 7 +- Cargo.toml | 4 +- examples/python-dataflow/README.md | 33 +++- examples/python-dataflow/dataflow.yml | 45 +++-- examples/python-dataflow/dataflow_dynamic.yml | 45 ++++- examples/python-dataflow/plot_dynamic.py | 71 ------- examples/python-dataflow/run.rs | 20 +- node-hub/README.md | 88 +++++++++ .../dora-record/Cargo.toml | 0 {nodes_hub => node-hub}/dora-record/README.md | 0 .../dora-record/src/main.rs | 0 {nodes_hub => node-hub}/dora-rerun/Cargo.toml | 0 {nodes_hub => node-hub}/dora-rerun/README.md | 0 .../dora-rerun/src/main.rs | 0 node-hub/opencv-plot/README.md | 80 ++++++++ node-hub/opencv-plot/main.py | 179 ++++++++++++++++++ node-hub/opencv-plot/pyproject.toml | 25 +++ node-hub/opencv-video-capture/README.md | 53 ++++++ node-hub/opencv-video-capture/main.py | 96 ++++++++++ node-hub/opencv-video-capture/pyproject.toml | 25 +++ node-hub/ultralytics-yolo/README.md | 66 +++++++ node-hub/ultralytics-yolo/main.py | 158 ++++++++++++++++ node-hub/ultralytics-yolo/pyproject.toml | 25 +++ nodes_hub/opencv-plot/plot.py | 73 ------- nodes_hub/opencv-plot/requirements.txt | 2 - .../opencv-video-capture/requirements.txt | 3 - .../opencv-video-capture/video_capture.py | 52 ----- nodes_hub/ultralytics-yolo/requirements.txt | 3 - nodes_hub/ultralytics-yolo/yolo.py | 50 ----- 29 files changed, 915 insertions(+), 288 deletions(-) delete mode 100644 examples/python-dataflow/plot_dynamic.py create mode 100644 node-hub/README.md rename {nodes_hub => node-hub}/dora-record/Cargo.toml (100%) rename {nodes_hub => node-hub}/dora-record/README.md (100%) rename {nodes_hub => node-hub}/dora-record/src/main.rs (100%) rename {nodes_hub => node-hub}/dora-rerun/Cargo.toml (100%) rename {nodes_hub => node-hub}/dora-rerun/README.md (100%) rename {nodes_hub => node-hub}/dora-rerun/src/main.rs (100%) create mode 100644 node-hub/opencv-plot/README.md create mode 100644 node-hub/opencv-plot/main.py create mode 100644 node-hub/opencv-plot/pyproject.toml create mode 100644 node-hub/opencv-video-capture/README.md create mode 100644 node-hub/opencv-video-capture/main.py create mode 100644 node-hub/opencv-video-capture/pyproject.toml create mode 100644 node-hub/ultralytics-yolo/README.md create mode 100644 node-hub/ultralytics-yolo/main.py create mode 100644 node-hub/ultralytics-yolo/pyproject.toml delete mode 100644 nodes_hub/opencv-plot/plot.py delete mode 100644 nodes_hub/opencv-plot/requirements.txt delete mode 100644 nodes_hub/opencv-video-capture/requirements.txt delete mode 100644 nodes_hub/opencv-video-capture/video_capture.py delete mode 100644 nodes_hub/ultralytics-yolo/requirements.txt delete mode 100644 nodes_hub/ultralytics-yolo/yolo.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3534c0d5..3946c50d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -318,14 +318,15 @@ jobs: cd test_python_project dora up dora list + dora build dataflow.yml dora start dataflow.yml --name ci-python-test --detach sleep 10 dora stop --name ci-python-test --grace-duration 5s - pip install "numpy<2.0.0" opencv-python + dora build ../examples/python-dataflow/dataflow_dynamic.yml dora start ../examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach - python ../examples/python-dataflow/plot_dynamic.py + ultralytics-yolo --name object-detection sleep 5 - dora stop --name ci-python-test --grace-duration 5s + dora stop --name ci-python-dynamic --grace-duration 5s dora destroy - name: "Test CLI (C)" diff --git a/Cargo.toml b/Cargo.toml index fa52a3b6..41fc4c5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,8 @@ members = [ "libraries/shared-memory-server", "libraries/extensions/download", "libraries/extensions/telemetry/*", - "nodes_hub/dora-record", - "nodes_hub/dora-rerun", + "node-hub/dora-record", + "node-hub/dora-rerun", "libraries/extensions/ros2-bridge", "libraries/extensions/ros2-bridge/msg-gen", "libraries/extensions/ros2-bridge/python", diff --git a/examples/python-dataflow/README.md b/examples/python-dataflow/README.md index 2211a721..8e2c75ec 100644 --- a/examples/python-dataflow/README.md +++ b/examples/python-dataflow/README.md @@ -1,25 +1,44 @@ # Python Dataflow Example -This examples shows how to create and connect dora operators and custom nodes in Python. +This examples shows how to create and connect dora nodes in Python. ## Overview The [`dataflow.yml`](./dataflow.yml) defines a simple dataflow graph with the following three nodes: - a webcam node, that connects to your webcam and feed the dataflow with webcam frame as jpeg compressed bytearray. -- an object detection node, that apply Yolo v5 on the webcam image. The model is imported from Pytorch Hub. The output is the bounding box of each object detected, the confidence and the class. You can have more info here: https://pytorch.org/hub/ultralytics_yolov5/ +- an object detection node, that apply Yolo v5 on the webcam image. The model is imported from Pytorch Hub. The output + is the bounding box of each object detected, the confidence and the associated label. You can have more info + here: https://pytorch.org/hub/ultralytics_yolov5/ - a window plotting node, that will retrieve the webcam image and the Yolov5 bounding box and join the two together. +The same dataflow is implemented for a `dynamic-node` in [`dataflow_dynamic.yml`](./dataflow_dynamic.yml). It contains +the same nodes as the previous dataflow, but the object detection node is a dynamic node. See the next section for more +information on how to start such a dataflow. + ## Getting started +After installing Rust, `dora-cli` and `Python >3.11`, you will need to **activate** (or create and **activate**) a +virtual +environment. Then, you will need to install the dependencies: + ```bash -cargo run --example python-dataflow +cd examples/python-dataflow +dora build ./dataflow.yml (or dora build ./dataflow_dynamic.yml) ``` -## Run the dataflow as a standalone +It will install the required dependencies for the Python nodes. -- Start the `dora-daemon`: +Then you can run the dataflow: +```bash +dora up +dora start ./dataflow.yml (or dora start ./dataflow_dynamic.yml) ``` -../../target/release/dora-daemon --run-dataflow dataflow.yml -``` + +**Note**: if you're running the dynamic dataflow, you will need to start manually the ultralytics-yolo node: + +```bash +# activate your virtual environment in another terminal +python ultralytics-yolo --name object-detection --model yolov5n.pt +``` \ No newline at end of file diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 2b608ce7..23562fed 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -1,25 +1,44 @@ nodes: - - id: webcam - build: pip install -r ../../nodes_hub/opencv-video-capture/requirements.txt - path: ../../nodes_hub/opencv-video-capture/video_capture.py + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture inputs: - tick: dora/timer/millis/50 + tick: plot/tick + outputs: - image + env: - DURATION: 100 + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 - - id: object_detection - build: pip install -r ../../nodes_hub/ultralytics-yolo/requirements.txt - path: ../../nodes_hub/ultralytics-yolo/yolo.py + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo inputs: - image: webcam/image + image: + source: camera/image + queue_size: 1 + outputs: - bbox + env: + MODEL: yolov5n.pt - id: plot - build: pip install -r ../../nodes_hub/opencv-plot/requirements.txt - path: ../../nodes_hub/opencv-plot/plot.py + build: pip install ../../node-hub/opencv-plot + path: opencv-plot inputs: - image: webcam/image - bbox: object_detection/bbox + image: + source: camera/image + queue_size: 1 + + bbox: object-detection/bbox + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 3c9ae1cb..140cd6c8 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -1,15 +1,44 @@ nodes: - - id: webcam - build: pip install -r ../../nodes_hub/opencv-webcam/requirements.txt - path: ../../nodes_hub/opencv-webcam/webcam.py + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture inputs: - tick: - source: dora/timer/millis/50 - queue_size: 1000 + tick: plot/tick + outputs: - image - - id: plot + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo path: dynamic inputs: - image: webcam/image + image: + source: camera/image + queue_size: 1 + + outputs: + - bbox + env: + MODEL: yolov5n.pt + + - id: plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + image: + source: camera/image + queue_size: 1 + + bbox: object-detection/bbox + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick \ No newline at end of file diff --git a/examples/python-dataflow/plot_dynamic.py b/examples/python-dataflow/plot_dynamic.py deleted file mode 100644 index 31c76ee2..00000000 --- a/examples/python-dataflow/plot_dynamic.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -from dataclasses import dataclass - -import cv2 -import numpy as np - -from dora import Node - -CI = os.environ.get("CI") - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) - -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -@dataclass -class Plotter: - frame: np.array = np.array([]) - bboxes: np.array = np.array([]) - - -if __name__ == "__main__": - plotter = Plotter() - node = Node("plot") - - for event in node: - event_type = event["type"] - if event_type == "INPUT": - if event["id"] == "image": - frame = event["value"].to_numpy() - frame = ( - event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - ) - plotter.frame = frame - - elif event["id"] == "bbox" and len(plotter.frame) != 0: - bboxs = event["value"].to_numpy() - plotter.bboxes = np.reshape(bboxs, (-1, 6)) - for bbox in plotter.bboxs: - [ - min_x, - min_y, - max_x, - max_y, - confidence, - label, - ] = bbox - cv2.rectangle( - plotter.frame, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - plotter.frame, - LABELS[int(label)] + f", {confidence:0.2f}", - (int(max_x), int(max_y)), - FONT, - 0.75, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", plotter.frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - break diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 20fdc399..25d0f870 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,4 +1,4 @@ -use dora_core::{get_pip_path, get_python_path, run}; +use dora_core::{get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; @@ -50,6 +50,14 @@ async fn main() -> eyre::Result<()> { ); } + run( + "pip", + &["install", "maturin"], + Some (venv), + ) + .await + .context("pip install maturin failed")?; + run( "maturin", &["develop"], @@ -66,6 +74,16 @@ async fn main() -> eyre::Result<()> { async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); + + // First build the dataflow (install requirements) + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--").arg("build").arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); diff --git a/node-hub/README.md b/node-hub/README.md new file mode 100644 index 00000000..9e59bc16 --- /dev/null +++ b/node-hub/README.md @@ -0,0 +1,88 @@ +## Dora Node Hub + +This hub contains useful pre-built nodes for Dora. + +# Structure + +The structure of the node hub is as follows (please use the same structure if you need to add a new node): + +``` +node-hub/ +└── your-node/ + ├── main.py + ├── README.mdr + └── pyproject.toml +``` + +The idea is to make a `pyproject.toml` file that will install the required dependencies for the node **and** attach main +function of the node inside a callable script in your environment. + +To do so, you will need to add a `main` function inside the `main.py` file. + +```python +def main(): + pass +``` + +And then you will need to adapt the following `pyproject.toml` file: + +```toml +[tool.poetry] +name = "[name of the node e.g. video-encoder, with '-' to replace spaces]" +version = "0.1" +authors = ["[Pseudo/Name] <[email]>"] +description = "Dora Node for []" +readme = "README.md" + +packages = [ + { include = "main.py", to = "[name of the node with '_' to replace spaces]" } +] + +[tool.poetry.dependencies] +python = "^3.11" +dora-rs = "0.3.5" +... [add your dependencies here] ... + +[tool.poetry.scripts] +[name of the node with '-' to replace spaces] = "[name of the node with '_' to replace spaces].main:main" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +``` + +Finally, the README.md file should explicit all inputs/outputs of the node and how to configure it in the YAML file. + +# Example + +```toml +[tool.poetry] +name = "opencv-plot" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for plotting data with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_plot" } +] + +[tool.poetry.dependencies] +python = "^3.11" +dora-rs = "^0.3.5" +opencv-python = "^4.10.0.84" + +[tool.poetry.scripts] +opencv-plot = "opencv_plot.main:main" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../NOTICE.md) for more information. diff --git a/nodes_hub/dora-record/Cargo.toml b/node-hub/dora-record/Cargo.toml similarity index 100% rename from nodes_hub/dora-record/Cargo.toml rename to node-hub/dora-record/Cargo.toml diff --git a/nodes_hub/dora-record/README.md b/node-hub/dora-record/README.md similarity index 100% rename from nodes_hub/dora-record/README.md rename to node-hub/dora-record/README.md diff --git a/nodes_hub/dora-record/src/main.rs b/node-hub/dora-record/src/main.rs similarity index 100% rename from nodes_hub/dora-record/src/main.rs rename to node-hub/dora-record/src/main.rs diff --git a/nodes_hub/dora-rerun/Cargo.toml b/node-hub/dora-rerun/Cargo.toml similarity index 100% rename from nodes_hub/dora-rerun/Cargo.toml rename to node-hub/dora-rerun/Cargo.toml diff --git a/nodes_hub/dora-rerun/README.md b/node-hub/dora-rerun/README.md similarity index 100% rename from nodes_hub/dora-rerun/README.md rename to node-hub/dora-rerun/README.md diff --git a/nodes_hub/dora-rerun/src/main.rs b/node-hub/dora-rerun/src/main.rs similarity index 100% rename from nodes_hub/dora-rerun/src/main.rs rename to node-hub/dora-rerun/src/main.rs diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md new file mode 100644 index 00000000..95017cbb --- /dev/null +++ b/node-hub/opencv-plot/README.md @@ -0,0 +1,80 @@ +# Dora Node for plotting data with OpenCV + +This node is used to plot a text and a list of bbox on a base image (ideal for object detection). + +# YAML + +```yaml + - id: opencv-plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + # image: Arrow array of size 1 containing the base image + # bbox: Arrow array of bbox + # text: Arrow array of size 1 containing the text to be plotted + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick + + env: + PLOT_WIDTH: 640 # optional, default is image input width + PLOT_HEIGHT: 480 # optional, default is image input height +``` + +# Inputs +- +- `tick`: empty Arrow array to trigger the capture + +- `image`: Arrow array containing the base image + +```python +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} + +``` + +- `text`: Arrow array containing the text to be plotted + +```python +text = { + "text": str, + "font_scale": np.float32, + "color": (np.uint8, np.uint8, np.uint8), + "thickness": np.uint32, + "position": (np.uint32, np.uint32) +} + +encoded_text = pa.array([text]) + +decoded_text = { + "text": encoded_text[0]["text"].as_py(), + "font_scale": np.float32(encoded_text[0]["font_scale"].as_py()), + "color": (np.uint8(encoded_text[0]["color"].as_py()[0]), + np.uint8(encoded_text[0]["color"].as_py()[1]), + np.uint8(encoded_text[0]["color"].as_py()[2])), + "thickness": np.uint32(encoded_text[0]["thickness"].as_py()), + "position": (np.uint32(encoded_text[0]["position"].as_py()[0]), + np.uint32(encoded_text[0]["position"].as_py()[1])) +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/main.py new file mode 100644 index 00000000..ec004701 --- /dev/null +++ b/node-hub/opencv-plot/main.py @@ -0,0 +1,179 @@ +import os +import argparse +import cv2 + +import numpy as np +import pyarrow as pa + +from dora import Node + + +class Plot: + frame: np.array = np.array([]) + + bboxes: {} = { + "bbox": np.array([]), + "conf": np.array([]), + "names": np.array([]), + } + + text: {} = { + "text": "", + "font_scale": np.float32(0.0), + "color": (np.uint8(0), np.uint8(0), np.uint8(0)), + "thickness": np.uint32(0), + "position": (np.uint32(0), np.uint32(0)), + } + + width: np.uint32 = None + height: np.uint32 = None + + +def plot_frame(plot, ci_enabled): + for bbox in zip(plot.bboxes["bbox"], plot.bboxes["conf"], plot.bboxes["names"]): + [ + [min_x, min_y, max_x, max_y], + confidence, + label, + ] = bbox + cv2.rectangle( + plot.frame, + (int(min_x), int(min_y)), + (int(max_x), int(max_y)), + (0, 255, 0), + 2, + ) + + cv2.putText( + plot.frame, + f"{label}, {confidence:0.2f}", + (int(max_x) - 120, int(max_y) - 10), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 0), + 1, + 1, + ) + + cv2.putText( + plot.frame, + plot.text["text"], + (int(plot.text["position"][0]), int(plot.text["position"][1])), + cv2.FONT_HERSHEY_SIMPLEX, + float(plot.text["font_scale"]), + (int(plot.text["color"][0]), int(plot.text["color"][1]), int(plot.text["color"][2])), + int(plot.text["thickness"]), + 1, + ) + + if plot.width is not None and plot.height is not None: + plot.frame = cv2.resize(plot.frame, (plot.width, plot.height)) + + if not ci_enabled: + if len(plot.frame.shape) >= 3: + cv2.imshow("Dora Node: opencv-plot", plot.frame) + + if cv2.waitKey(1) & 0xFF == ord('q'): + return True + + return False + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="OpenCV Plotter: This node is used to plot text and bounding boxes on an image.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="opencv-plot") + parser.add_argument("--plot-width", type=int, required=False, help="The width of the plot.", default=None) + parser.add_argument("--plot-height", type=int, required=False, help="The height of the plot.", default=None) + + args = parser.parse_args() + + plot_width = os.getenv("PLOT_WIDTH", args.plot_width) + plot_height = os.getenv("PLOT_HEIGHT", args.plot_height) + + if plot_width is not None: + if isinstance(plot_width, str) and plot_width.isnumeric(): + plot_width = int(plot_width) + + if plot_height is not None: + if isinstance(plot_height, str) and plot_height.isnumeric(): + plot_height = int(plot_height) + + # check if the code is running in a CI environment (e.g. GitHub Actions) (parse to bool) + ci_enabled = os.getenv("CI", False) + if ci_enabled == "true": + ci_enabled = True + + node = Node(args.name) # provide the name to connect to the dataflow if dynamic node + plot = Plot() + + plot.width = plot_width + plot.height = plot_height + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "tick": + node.send_output( + "tick", + pa.array([]), + event["metadata"] + ) + + elif event_id == "image": + arrow_image = event["value"][0] + image = { + "width": np.uint32(arrow_image["width"].as_py()), + "height": np.uint32(arrow_image["height"].as_py()), + "channels": np.uint8(arrow_image["channels"].as_py()), + "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + } + + plot.frame = np.reshape(image["data"], (image["height"], image["width"], image["channels"])) + + if plot_frame(plot, ci_enabled): + break + + elif event_id == "bbox": + arrow_bbox = event["value"][0] + plot.bboxes = { + "bbox": arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4), + "conf": arrow_bbox["conf"].values.to_numpy(), + "names": arrow_bbox["names"].values.to_pylist(), + } + + if plot_frame(plot, ci_enabled): + break + + elif event_id == "text": + arrow_text = event["value"][0] + plot.text = { + "text": arrow_text["text"].as_py(), + "font_scale": np.float32(arrow_text["font_scale"].as_py()), + "color": (np.uint8(arrow_text["color"].as_py()[0]), + np.uint8(arrow_text["color"].as_py()[1]), + np.uint8(arrow_text["color"].as_py()[2])), + "thickness": np.uint32(arrow_text["thickness"].as_py()), + "position": (np.uint32(arrow_text["position"].as_py()[0]), + np.uint32(arrow_text["position"].as_py()[1])) + } + + if plot_frame(plot, ci_enabled): + break + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/opencv-plot/pyproject.toml b/node-hub/opencv-plot/pyproject.toml new file mode 100644 index 00000000..a1f01c45 --- /dev/null +++ b/node-hub/opencv-plot/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "opencv-plot" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for plotting text and bbox on image with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_plot" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +opencv-python = ">= 4.1.1" + +[tool.poetry.scripts] +opencv-plot = "opencv_plot.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md new file mode 100644 index 00000000..f4f10cdd --- /dev/null +++ b/node-hub/opencv-video-capture/README.md @@ -0,0 +1,53 @@ +# Dora Node for capturing video with OpenCV + +This node is used to capture video from a camera using OpenCV. + +# YAML + +```yaml + - id: opencv-video-capture + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: dora/timer/millis/16 # try to capture at 60fps + + outputs: + - image: # the captured image + + env: + PATH: 0 # optional, default is 0 + + IMAGE_WIDTH: 640 # optional, default is video capture width + IMAGE_HEIGHT: 480 # optional, default is video capture height +``` + +# Inputs + +- `tick`: empty Arrow array to trigger the capture + +# Outputs + +- `image`: an arrow array containing the captured image + +```Python + +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py new file mode 100644 index 00000000..5160e0fb --- /dev/null +++ b/node-hub/opencv-video-capture/main.py @@ -0,0 +1,96 @@ +import os +import argparse +import cv2 + +import numpy as np +import pyarrow as pa + +from dora import Node + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="OpenCV Video Capture: This node is used to capture video from a camera.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="opencv-video-capture") + parser.add_argument("--path", type=int, required=False, + help="The path of the device to capture (e.g. /dev/video1, or an index like 0, 1...", default=0) + parser.add_argument("--image-width", type=int, required=False, + help="The width of the image output. Default is the camera width.", default=None) + parser.add_argument("--image-height", type=int, required=False, + help="The height of the camera. Default is the camera height.", default=None) + + args = parser.parse_args() + + video_capture_path = os.getenv("CAPTURE_PATH", args.path) + + if isinstance(video_capture_path, str) and video_capture_path.isnumeric(): + video_capture_path = int(video_capture_path) + + print(type(video_capture_path)) + + image_width = os.getenv("IMAGE_WIDTH", args.image_width) + image_height = os.getenv("IMAGE_HEIGHT", args.image_height) + + if image_width is not None: + if isinstance(image_width, str) and image_width.isnumeric(): + image_width = int(image_width) + + if image_height is not None: + if isinstance(image_height, str) and image_height.isnumeric(): + image_height = int(image_height) + + video_capture = cv2.VideoCapture(video_capture_path) + node = Node(args.name) + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "tick": + ret, frame = video_capture.read() + + if not ret: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + cv2.putText( + frame, + f'Error: Could not read frame from camera at path {video_capture_path}.', + (int(30), int(30)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.50, + (255, 255, 255), + 1, + 1, + ) + + # resize the frame + if image_width is not None and image_height is not None: + frame = cv2.resize(frame, (image_width, image_height)) + + image = { + "width": np.uint32(frame.shape[1]), + "height": np.uint32(frame.shape[0]), + "channels": np.uint8(frame.shape[2]), + "data": frame.ravel() + } + + node.send_output( + "image", + pa.array([image]), + event["metadata"] + ) + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/opencv-video-capture/pyproject.toml b/node-hub/opencv-video-capture/pyproject.toml new file mode 100644 index 00000000..00100acf --- /dev/null +++ b/node-hub/opencv-video-capture/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "opencv-video-capture" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for capturing video with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_video_capture" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +opencv-python = ">= 4.1.1" + +[tool.poetry.scripts] +opencv-video-capture = "opencv_video_capture.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/node-hub/ultralytics-yolo/README.md b/node-hub/ultralytics-yolo/README.md new file mode 100644 index 00000000..5a66c832 --- /dev/null +++ b/node-hub/ultralytics-yolo/README.md @@ -0,0 +1,66 @@ +# Dora Node for detecting objects in images using YOLOv8 + +This node is used to detect objects in images using YOLOv8. + +# YAML + +```yaml + - id: object_detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: webcam/image + + outputs: + - bbox + env: + MODEL: yolov5n.pt +``` + +# Inputs + +- `image`: Arrow array containing the base image + +```python +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} + +``` + +# Outputs + +- `bbox`: an arrow array containing the bounding boxes, confidence scores, and class names of the detected objects + +```Python + +bbox = { + "bbox": np.array, # flattened array of bounding boxes + "conf": np.array, # flat array of confidence scores + "names": np.array, # flat array of class names +} + +encoded_bbox = pa.array([bbox]) + +decoded_bbox = { + "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), + "conf": encoded_bbox[0]["conf"].values.to_numpy(), + "names": encoded_bbox[0]["names"].values.to_pylist(), +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/main.py new file mode 100644 index 00000000..9f4d9b15 --- /dev/null +++ b/node-hub/ultralytics-yolo/main.py @@ -0,0 +1,158 @@ +import os +import argparse + +import numpy as np +import pyarrow as pa + +from dora import Node +from ultralytics import YOLO + +LABELS = [ + "ABC", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +] + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="UltraLytics YOLO: This node is used to perform object detection using the UltraLytics YOLO model.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="ultralytics-yolo") + parser.add_argument("--model", type=str, required=False, + help="The name of the model file (e.g. yolov8n.pt).", default="yolov8n.pt") + + args = parser.parse_args() + + model_path = os.getenv("MODEL", args.model) + + model = YOLO(model_path) + node = Node(args.name) + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "image": + arrow_image = event["value"][0] + image = { + "width": np.uint32(arrow_image["width"].as_py()), + "height": np.uint32(arrow_image["height"].as_py()), + "channels": np.uint8(arrow_image["channels"].as_py()), + "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + } + + frame = image["data"].reshape((image["height"], image["width"], 3)) + + frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + results = model(frame, verbose=False) # includes NMS + + bboxes = np.array(results[0].boxes.xyxy.cpu()) + conf = np.array(results[0].boxes.conf.cpu()) + labels = np.array(results[0].boxes.cls.cpu()) + + names = [LABELS[int(label)] for label in labels] + + bbox = { + "bbox": bboxes.ravel(), + "conf": conf, + "names": names, + } + + node.send_output( + "bbox", + pa.array([bbox]), + event["metadata"], + ) + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/ultralytics-yolo/pyproject.toml b/node-hub/ultralytics-yolo/pyproject.toml new file mode 100644 index 00000000..0d7feacf --- /dev/null +++ b/node-hub/ultralytics-yolo/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "ultralytics-yolo" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for object detection with Ultralytics YOLOv8" +readme = "README.md" + +packages = [ + { include = "main.py", to = "ultralytics_yolo" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +ultralytics = "<= 8.2.52" + +[tool.poetry.scripts] +ultralytics-yolo = "ultralytics_yolo.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/nodes_hub/opencv-plot/plot.py b/nodes_hub/opencv-plot/plot.py deleted file mode 100644 index d2c4ce74..00000000 --- a/nodes_hub/opencv-plot/plot.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from dataclasses import dataclass - -import cv2 -import numpy as np - -from dora import Node - -CI = os.environ.get("CI") - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) - -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -@dataclass -class Plotter: - frame: np.array = np.array([]) - bboxes: np.array = np.array([[]]) - conf: np.array = np.array([]) - label: np.array = np.array([]) - - -if __name__ == "__main__": - plotter = Plotter() - node = Node() - for event in node: - event_type = event["type"] - if event_type == "INPUT": - if event["id"] == "image": - frame = event["value"].to_numpy() - frame = frame.reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)).copy() - plotter.frame = frame - - elif event["id"] == "bbox": - bboxes = event["value"][0]["bbox"].values.to_numpy() - conf = event["value"][0]["conf"].values.to_numpy() - label = event["value"][0]["names"].values.to_pylist() - plotter.bboxes = np.reshape(bboxes, (-1, 4)) - plotter.conf = conf - plotter.label = label - continue - - for bbox in zip(plotter.bboxes, plotter.conf, plotter.label): - [ - [min_x, min_y, max_x, max_y], - confidence, - label, - ] = bbox - cv2.rectangle( - plotter.frame, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - plotter.frame, - f"{label}, {confidence:0.2f}", - (int(max_x) - 120, int(max_y) - 10), - FONT, - 0.5, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", plotter.frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - break diff --git a/nodes_hub/opencv-plot/requirements.txt b/nodes_hub/opencv-plot/requirements.txt deleted file mode 100644 index 7f3ef381..00000000 --- a/nodes_hub/opencv-plot/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -numpy<2.0.0 -opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/requirements.txt b/nodes_hub/opencv-video-capture/requirements.txt deleted file mode 100644 index 1db957a5..00000000 --- a/nodes_hub/opencv-video-capture/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy<2.0.0 -pyarrow -opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/video_capture.py b/nodes_hub/opencv-video-capture/video_capture.py deleted file mode 100644 index cad4320c..00000000 --- a/nodes_hub/opencv-video-capture/video_capture.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import time - -import cv2 -import numpy as np -import pyarrow as pa - -from dora import Node - -CAM_INDEX = int(os.getenv("CAM_INDEX", "0")) -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) -MAX_DURATION = int(os.getenv("DURATION", "20")) -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -start = time.time() - - -if __name__ == "__main__": - - video_capture = cv2.VideoCapture(CAM_INDEX) - node = Node() - - while time.time() - start < MAX_DURATION: - event = node.next() - if event is None: - break - if event is not None: - event_type = event["type"] - if event_type == "INPUT": - ret, frame = video_capture.read() - - # Fail to read camera - if not ret: - frame = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3), dtype=np.uint8) - cv2.putText( - frame, - "No Webcam was found at index %d" % (CAM_INDEX), - (int(30), int(30)), - FONT, - 0.75, - (255, 255, 255), - 2, - 1, - ) - - node.send_output( - "image", - pa.array(frame.ravel()), - event["metadata"], - ) diff --git a/nodes_hub/ultralytics-yolo/requirements.txt b/nodes_hub/ultralytics-yolo/requirements.txt deleted file mode 100644 index d8872ab6..00000000 --- a/nodes_hub/ultralytics-yolo/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy<2.0.0 -pyarrow -ultralytics \ No newline at end of file diff --git a/nodes_hub/ultralytics-yolo/yolo.py b/nodes_hub/ultralytics-yolo/yolo.py deleted file mode 100644 index 392c3ee9..00000000 --- a/nodes_hub/ultralytics-yolo/yolo.py +++ /dev/null @@ -1,50 +0,0 @@ -## Imports -import os - -import numpy as np -import pyarrow as pa -from ultralytics import YOLO - -from dora import Node - -## OS Environment variable -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) -MODEL = os.getenv("MODEL", "yolov8n.pt") - -if __name__ == "__main__": - - model = YOLO(MODEL) - - node = Node("object_detection") - - for event in node: - event_type = event["type"] - if event_type == "INPUT": - event_id = event["id"] - if event_id == "image": - frame = ( - event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - ) - frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) - results = model(frame, verbose=False) # includes NMS - # Process results - bboxes = np.array(results[0].boxes.xyxy.cpu()) - conf = np.array(results[0].boxes.conf.cpu()) - labels = np.array(results[0].boxes.cls.cpu()) - names = [model.names.get(label) for label in labels] - - node.send_output( - "bbox", - pa.array( - [ - { - "bbox": bboxes.ravel(), - "conf": conf, - "labels": labels, - "names": names, - } - ] - ), - event["metadata"], - ) From 2b40fe63b423b77b5121ef9e1a89d602d1827f72 Mon Sep 17 00:00:00 2001 From: Hennzau Date: Wed, 17 Jul 2024 11:21:55 +0200 Subject: [PATCH 05/18] Separate a simple dataflow for CI (without yolo) and a yolo dataflow --- .github/workflows/ci.yml | 2 +- examples/python-dataflow/README.md | 9 ++-- examples/python-dataflow/dataflow.yml | 15 ------- examples/python-dataflow/dataflow_dynamic.yml | 17 +------ examples/python-dataflow/dataflow_yolo.yml | 44 +++++++++++++++++++ examples/python-dataflow/run.rs | 10 ++--- 6 files changed, 52 insertions(+), 45 deletions(-) create mode 100644 examples/python-dataflow/dataflow_yolo.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3946c50d..faee5960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,7 +324,7 @@ jobs: dora stop --name ci-python-test --grace-duration 5s dora build ../examples/python-dataflow/dataflow_dynamic.yml dora start ../examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach - ultralytics-yolo --name object-detection + opencv-plot --name plot sleep 5 dora stop --name ci-python-dynamic --grace-duration 5s dora destroy diff --git a/examples/python-dataflow/README.md b/examples/python-dataflow/README.md index 8e2c75ec..aa0c9be0 100644 --- a/examples/python-dataflow/README.md +++ b/examples/python-dataflow/README.md @@ -7,13 +7,10 @@ This examples shows how to create and connect dora nodes in Python. The [`dataflow.yml`](./dataflow.yml) defines a simple dataflow graph with the following three nodes: - a webcam node, that connects to your webcam and feed the dataflow with webcam frame as jpeg compressed bytearray. -- an object detection node, that apply Yolo v5 on the webcam image. The model is imported from Pytorch Hub. The output - is the bounding box of each object detected, the confidence and the associated label. You can have more info - here: https://pytorch.org/hub/ultralytics_yolov5/ -- a window plotting node, that will retrieve the webcam image and the Yolov5 bounding box and join the two together. +- a window plotting node, that will retrieve the webcam image and plot it. The same dataflow is implemented for a `dynamic-node` in [`dataflow_dynamic.yml`](./dataflow_dynamic.yml). It contains -the same nodes as the previous dataflow, but the object detection node is a dynamic node. See the next section for more +the same nodes as the previous dataflow, but the plot node is a dynamic node. See the next section for more information on how to start such a dataflow. ## Getting started @@ -40,5 +37,5 @@ dora start ./dataflow.yml (or dora start ./dataflow_dynamic.yml) ```bash # activate your virtual environment in another terminal -python ultralytics-yolo --name object-detection --model yolov5n.pt +python opencv-plot --name plot ``` \ No newline at end of file diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 23562fed..09015863 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -13,19 +13,6 @@ nodes: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - - id: object-detection - build: pip install ../../node-hub/ultralytics-yolo - path: ultralytics-yolo - inputs: - image: - source: camera/image - queue_size: 1 - - outputs: - - bbox - env: - MODEL: yolov5n.pt - - id: plot build: pip install ../../node-hub/opencv-plot path: opencv-plot @@ -34,8 +21,6 @@ nodes: source: camera/image queue_size: 1 - bbox: object-detection/bbox - tick: source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph queue_size: 1 diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 140cd6c8..b949d0a4 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -13,29 +13,14 @@ nodes: IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 - - id: object-detection - build: pip install ../../node-hub/ultralytics-yolo - path: dynamic - inputs: - image: - source: camera/image - queue_size: 1 - - outputs: - - bbox - env: - MODEL: yolov5n.pt - - id: plot build: pip install ../../node-hub/opencv-plot - path: opencv-plot + path: dynamic inputs: image: source: camera/image queue_size: 1 - bbox: object-detection/bbox - tick: source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph queue_size: 1 diff --git a/examples/python-dataflow/dataflow_yolo.yml b/examples/python-dataflow/dataflow_yolo.yml new file mode 100644 index 00000000..23562fed --- /dev/null +++ b/examples/python-dataflow/dataflow_yolo.yml @@ -0,0 +1,44 @@ +nodes: + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: plot/tick + + outputs: + - image + + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: + source: camera/image + queue_size: 1 + + outputs: + - bbox + env: + MODEL: yolov5n.pt + + - id: plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + image: + source: camera/image + queue_size: 1 + + bbox: object-detection/bbox + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick \ No newline at end of file diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 25d0f870..1271776b 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -50,13 +50,9 @@ async fn main() -> eyre::Result<()> { ); } - run( - "pip", - &["install", "maturin"], - Some (venv), - ) - .await - .context("pip install maturin failed")?; + run("pip", &["install", "maturin"], Some(venv)) + .await + .context("pip install maturin failed")?; run( "maturin", From 699f460c3bc01c48002927b8ad4c43c1db9af623 Mon Sep 17 00:00:00 2001 From: Hennzau Date: Wed, 17 Jul 2024 15:07:37 +0200 Subject: [PATCH 06/18] fix pip get path + fix CI for dataflow.yml --- examples/python-dataflow/dataflow.yml | 14 +-- examples/python-dataflow/dataflow_dynamic.yml | 12 +-- examples/python-dataflow/dataflow_yolo.yml | 6 -- examples/python-dataflow/run.rs | 12 ++- node-hub/opencv-plot/main.py | 35 +++----- node-hub/opencv-video-capture/main.py | 4 - node-hub/ultralytics-yolo/main.py | 88 +------------------ 7 files changed, 25 insertions(+), 146 deletions(-) diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 09015863..fe0e6733 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -4,10 +4,9 @@ nodes: path: opencv-video-capture inputs: tick: plot/tick - outputs: - image - + - text env: CAPTURE_PATH: 0 IMAGE_WIDTH: 640 @@ -17,13 +16,8 @@ nodes: build: pip install ../../node-hub/opencv-plot path: opencv-plot inputs: - image: - source: camera/image - queue_size: 1 - - tick: - source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph - queue_size: 1 - + image: camera/image + tick: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + text: camera/text outputs: - tick \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index b949d0a4..7b948d5f 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -4,10 +4,8 @@ nodes: path: opencv-video-capture inputs: tick: plot/tick - outputs: - image - env: CAPTURE_PATH: 0 IMAGE_WIDTH: 640 @@ -17,13 +15,7 @@ nodes: build: pip install ../../node-hub/opencv-plot path: dynamic inputs: - image: - source: camera/image - queue_size: 1 - - tick: - source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph - queue_size: 1 - + image: camera/image + tick: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph outputs: - tick \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_yolo.yml b/examples/python-dataflow/dataflow_yolo.yml index 23562fed..1b4f75d3 100644 --- a/examples/python-dataflow/dataflow_yolo.yml +++ b/examples/python-dataflow/dataflow_yolo.yml @@ -4,10 +4,8 @@ nodes: path: opencv-video-capture inputs: tick: plot/tick - outputs: - image - env: CAPTURE_PATH: 0 IMAGE_WIDTH: 640 @@ -20,7 +18,6 @@ nodes: image: source: camera/image queue_size: 1 - outputs: - bbox env: @@ -33,12 +30,9 @@ nodes: image: source: camera/image queue_size: 1 - bbox: object-detection/bbox - tick: source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph queue_size: 1 - outputs: - tick \ No newline at end of file diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 1271776b..b575234d 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,4 +1,4 @@ -use dora_core::{get_python_path, run}; +use dora_core::{get_pip_path, get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; @@ -50,9 +50,13 @@ async fn main() -> eyre::Result<()> { ); } - run("pip", &["install", "maturin"], Some(venv)) - .await - .context("pip install maturin failed")?; + run( + get_pip_path().context("Could not get pip binary")?, + &["install", "maturin"], + Some(venv), + ) + .await + .context("pip install maturin failed")?; run( "maturin", diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/main.py index ec004701..1a65cd5d 100644 --- a/node-hub/opencv-plot/main.py +++ b/node-hub/opencv-plot/main.py @@ -17,13 +17,7 @@ class Plot: "names": np.array([]), } - text: {} = { - "text": "", - "font_scale": np.float32(0.0), - "color": (np.uint8(0), np.uint8(0), np.uint8(0)), - "thickness": np.uint32(0), - "position": (np.uint32(0), np.uint32(0)), - } + text: str = "" width: np.uint32 = None height: np.uint32 = None @@ -57,12 +51,12 @@ def plot_frame(plot, ci_enabled): cv2.putText( plot.frame, - plot.text["text"], - (int(plot.text["position"][0]), int(plot.text["position"][1])), + plot.text, + (20, 20), cv2.FONT_HERSHEY_SIMPLEX, - float(plot.text["font_scale"]), - (int(plot.text["color"][0]), int(plot.text["color"][1]), int(plot.text["color"][2])), - int(plot.text["thickness"]), + 0.5, + (255, 255, 255), + 1, 1, ) @@ -122,6 +116,9 @@ def main(): event_id = event["id"] if event_id == "tick": + if ci_enabled: + break + node.send_output( "tick", pa.array([]), @@ -154,23 +151,11 @@ def main(): break elif event_id == "text": - arrow_text = event["value"][0] - plot.text = { - "text": arrow_text["text"].as_py(), - "font_scale": np.float32(arrow_text["font_scale"].as_py()), - "color": (np.uint8(arrow_text["color"].as_py()[0]), - np.uint8(arrow_text["color"].as_py()[1]), - np.uint8(arrow_text["color"].as_py()[2])), - "thickness": np.uint32(arrow_text["thickness"].as_py()), - "position": (np.uint32(arrow_text["position"].as_py()[0]), - np.uint32(arrow_text["position"].as_py()[1])) - } + plot.text = event["value"][0].as_py() if plot_frame(plot, ci_enabled): break - elif event_type == "STOP": - break elif event_type == "ERROR": raise Exception(event["error"]) diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py index 5160e0fb..0b2d22f5 100644 --- a/node-hub/opencv-video-capture/main.py +++ b/node-hub/opencv-video-capture/main.py @@ -29,8 +29,6 @@ def main(): if isinstance(video_capture_path, str) and video_capture_path.isnumeric(): video_capture_path = int(video_capture_path) - print(type(video_capture_path)) - image_width = os.getenv("IMAGE_WIDTH", args.image_width) image_height = os.getenv("IMAGE_HEIGHT", args.image_height) @@ -86,8 +84,6 @@ def main(): event["metadata"] ) - elif event_type == "STOP": - break elif event_type == "ERROR": raise Exception(event["error"]) diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/main.py index 9f4d9b15..2bd3ce00 100644 --- a/node-hub/ultralytics-yolo/main.py +++ b/node-hub/ultralytics-yolo/main.py @@ -7,90 +7,6 @@ import pyarrow as pa from dora import Node from ultralytics import YOLO -LABELS = [ - "ABC", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "backpack", - "umbrella", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "dining table", - "toilet", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush", -] - - def main(): # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. parser = argparse.ArgumentParser( @@ -134,7 +50,7 @@ def main(): conf = np.array(results[0].boxes.conf.cpu()) labels = np.array(results[0].boxes.cls.cpu()) - names = [LABELS[int(label)] for label in labels] + names = [model.names.get(label) for label in labels] bbox = { "bbox": bboxes.ravel(), @@ -148,8 +64,6 @@ def main(): event["metadata"], ) - elif event_type == "STOP": - break elif event_type == "ERROR": raise Exception(event["error"]) From 0b0e9aeeffcc851e11e62c51b941973fd423cfe5 Mon Sep 17 00:00:00 2001 From: Hennzau Date: Thu, 18 Jul 2024 18:43:39 +0200 Subject: [PATCH 07/18] Fix typos, CI and README --- examples/python-dataflow/dataflow.yml | 6 +- examples/python-dataflow/dataflow_dynamic.yml | 6 +- examples/python-dataflow/dataflow_yolo.yml | 10 ++-- node-hub/opencv-plot/README.md | 48 +++++++-------- node-hub/opencv-plot/main.py | 60 +++++++++---------- node-hub/opencv-video-capture/README.md | 4 +- node-hub/opencv-video-capture/main.py | 6 +- node-hub/ultralytics-yolo/README.md | 6 +- node-hub/ultralytics-yolo/main.py | 2 +- 9 files changed, 73 insertions(+), 75 deletions(-) diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index fe0e6733..e6a743ab 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -3,7 +3,8 @@ nodes: build: pip install ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: plot/tick + tick: dora/timer/millis/16 + stop: plot/end outputs: - image - text @@ -17,7 +18,6 @@ nodes: path: opencv-plot inputs: image: camera/image - tick: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph text: camera/text outputs: - - tick \ No newline at end of file + - end \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 7b948d5f..4253e2d8 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -3,7 +3,8 @@ nodes: build: pip install ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: plot/tick + tick: dora/timer/millis/16 + stop: plot/end outputs: - image env: @@ -16,6 +17,5 @@ nodes: path: dynamic inputs: image: camera/image - tick: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph outputs: - - tick \ No newline at end of file + - end \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_yolo.yml b/examples/python-dataflow/dataflow_yolo.yml index 1b4f75d3..e5f8ca48 100644 --- a/examples/python-dataflow/dataflow_yolo.yml +++ b/examples/python-dataflow/dataflow_yolo.yml @@ -3,7 +3,8 @@ nodes: build: pip install ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: plot/tick + tick: dora/timer/millis/16 + stop: plot/end outputs: - image env: @@ -21,7 +22,7 @@ nodes: outputs: - bbox env: - MODEL: yolov5n.pt + MODEL: yolov8n.pt - id: plot build: pip install ../../node-hub/opencv-plot @@ -31,8 +32,5 @@ nodes: source: camera/image queue_size: 1 bbox: object-detection/bbox - tick: - source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph - queue_size: 1 outputs: - - tick \ No newline at end of file + - end \ No newline at end of file diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md index 95017cbb..8f6cdef1 100644 --- a/node-hub/opencv-plot/README.md +++ b/node-hub/opencv-plot/README.md @@ -13,12 +13,8 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o # bbox: Arrow array of bbox # text: Arrow array of size 1 containing the text to be plotted - tick: - source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph - queue_size: 1 - outputs: - - tick + - end env: PLOT_WIDTH: 640 # optional, default is image input width @@ -26,13 +22,11 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o ``` # Inputs -- -- `tick`: empty Arrow array to trigger the capture - `image`: Arrow array containing the base image ```python -image = { +image: { "width": np.uint32, "height": np.uint32, "channels": np.uint8, @@ -50,29 +44,33 @@ decoded_image = { ``` +- `bbox`: an arrow array containing the bounding boxes, confidence scores, and class names of the detected objects + +```Python + +bbox: { + "bbox": np.array, # flattened array of bounding boxes + "conf": np.array, # flat array of confidence scores + "names": np.array, # flat array of class names +} + +encoded_bbox = pa.array([bbox]) + +decoded_bbox = { + "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), + "conf": encoded_bbox[0]["conf"].values.to_numpy(), + "names": encoded_bbox[0]["names"].values.to_numpy(zero_copy_only=False), +} +``` + - `text`: Arrow array containing the text to be plotted ```python -text = { - "text": str, - "font_scale": np.float32, - "color": (np.uint8, np.uint8, np.uint8), - "thickness": np.uint32, - "position": (np.uint32, np.uint32) -} +text: str encoded_text = pa.array([text]) -decoded_text = { - "text": encoded_text[0]["text"].as_py(), - "font_scale": np.float32(encoded_text[0]["font_scale"].as_py()), - "color": (np.uint8(encoded_text[0]["color"].as_py()[0]), - np.uint8(encoded_text[0]["color"].as_py()[1]), - np.uint8(encoded_text[0]["color"].as_py()[2])), - "thickness": np.uint32(encoded_text[0]["thickness"].as_py()), - "position": (np.uint32(encoded_text[0]["position"].as_py()[0]), - np.uint32(encoded_text[0]["position"].as_py()[1])) -} +decoded_text = encoded_text[0].as_py() ``` ## License diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/main.py index 1a65cd5d..0648c3fd 100644 --- a/node-hub/opencv-plot/main.py +++ b/node-hub/opencv-plot/main.py @@ -7,6 +7,8 @@ import pyarrow as pa from dora import Node +RUNNER_CI = True if os.getenv("CI", False) == "true" else False + class Plot: frame: np.array = np.array([]) @@ -23,7 +25,7 @@ class Plot: height: np.uint32 = None -def plot_frame(plot, ci_enabled): +def plot_frame(plot): for bbox in zip(plot.bboxes["bbox"], plot.bboxes["conf"], plot.bboxes["names"]): [ [min_x, min_y, max_x, max_y], @@ -63,15 +65,10 @@ def plot_frame(plot, ci_enabled): if plot.width is not None and plot.height is not None: plot.frame = cv2.resize(plot.frame, (plot.width, plot.height)) - if not ci_enabled: + if not RUNNER_CI: if len(plot.frame.shape) >= 3: cv2.imshow("Dora Node: opencv-plot", plot.frame) - if cv2.waitKey(1) & 0xFF == ord('q'): - return True - - return False - def main(): # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. @@ -96,11 +93,6 @@ def main(): if isinstance(plot_height, str) and plot_height.isnumeric(): plot_height = int(plot_height) - # check if the code is running in a CI environment (e.g. GitHub Actions) (parse to bool) - ci_enabled = os.getenv("CI", False) - if ci_enabled == "true": - ci_enabled = True - node = Node(args.name) # provide the name to connect to the dataflow if dynamic node plot = Plot() @@ -115,17 +107,7 @@ def main(): if event_type == "INPUT": event_id = event["id"] - if event_id == "tick": - if ci_enabled: - break - - node.send_output( - "tick", - pa.array([]), - event["metadata"] - ) - - elif event_id == "image": + if event_id == "image": arrow_image = event["value"][0] image = { "width": np.uint32(arrow_image["width"].as_py()), @@ -136,29 +118,45 @@ def main(): plot.frame = np.reshape(image["data"], (image["height"], image["width"], image["channels"])) - if plot_frame(plot, ci_enabled): - break + plot_frame(plot) + if not RUNNER_CI: + if cv2.waitKey(1) & 0xFF == ord("q"): + break + else: + break # break the loop for CI elif event_id == "bbox": arrow_bbox = event["value"][0] plot.bboxes = { "bbox": arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4), "conf": arrow_bbox["conf"].values.to_numpy(), - "names": arrow_bbox["names"].values.to_pylist(), + "names": arrow_bbox["names"].values.to_numpy(zero_copy_only=False), } - if plot_frame(plot, ci_enabled): - break - + plot_frame(plot) + if not RUNNER_CI: + if cv2.waitKey(1) & 0xFF == ord("q"): + break + else: + break # break the loop for CI elif event_id == "text": plot.text = event["value"][0].as_py() - if plot_frame(plot, ci_enabled): - break + plot_frame(plot) + if not RUNNER_CI: + if cv2.waitKey(1) & 0xFF == ord("q"): + break + else: + break # break the loop for CI elif event_type == "ERROR": raise Exception(event["error"]) + node.send_output( + "end", + pa.array([0], type=pa.uint8()) + ) + if __name__ == "__main__": main() diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md index f4f10cdd..9aa2dbb8 100644 --- a/node-hub/opencv-video-capture/README.md +++ b/node-hub/opencv-video-capture/README.md @@ -10,7 +10,7 @@ This node is used to capture video from a camera using OpenCV. path: opencv-video-capture inputs: tick: dora/timer/millis/16 # try to capture at 60fps - + # stop: some stop signal from another node outputs: - image: # the captured image @@ -31,7 +31,7 @@ This node is used to capture video from a camera using OpenCV. ```Python -image = { +image: { "width": np.uint32, "height": np.uint32, "channels": np.uint8, diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py index 0b2d22f5..85a8e2c8 100644 --- a/node-hub/opencv-video-capture/main.py +++ b/node-hub/opencv-video-capture/main.py @@ -58,7 +58,7 @@ def main(): frame = np.zeros((480, 640, 3), dtype=np.uint8) cv2.putText( frame, - f'Error: Could not read frame from camera at path {video_capture_path}.', + f'Error: no frame for camera at path {video_capture_path}.', (int(30), int(30)), cv2.FONT_HERSHEY_SIMPLEX, 0.50, @@ -84,6 +84,10 @@ def main(): event["metadata"] ) + if event_id == "stop": + video_capture.release() + break + elif event_type == "ERROR": raise Exception(event["error"]) diff --git a/node-hub/ultralytics-yolo/README.md b/node-hub/ultralytics-yolo/README.md index 5a66c832..8a7bc8f2 100644 --- a/node-hub/ultralytics-yolo/README.md +++ b/node-hub/ultralytics-yolo/README.md @@ -22,7 +22,7 @@ This node is used to detect objects in images using YOLOv8. - `image`: Arrow array containing the base image ```python -image = { +image: { "width": np.uint32, "height": np.uint32, "channels": np.uint8, @@ -46,7 +46,7 @@ decoded_image = { ```Python -bbox = { +bbox: { "bbox": np.array, # flattened array of bounding boxes "conf": np.array, # flat array of confidence scores "names": np.array, # flat array of class names @@ -57,7 +57,7 @@ encoded_bbox = pa.array([bbox]) decoded_bbox = { "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), "conf": encoded_bbox[0]["conf"].values.to_numpy(), - "names": encoded_bbox[0]["names"].values.to_pylist(), + "names": encoded_bbox[0]["names"].values.to_numpy(zero_copy_only=False), } ``` diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/main.py index 2bd3ce00..fa3e48ba 100644 --- a/node-hub/ultralytics-yolo/main.py +++ b/node-hub/ultralytics-yolo/main.py @@ -41,7 +41,7 @@ def main(): "data": arrow_image["data"].values.to_numpy().astype(np.uint8) } - frame = image["data"].reshape((image["height"], image["width"], 3)) + frame = image["data"].reshape((image["height"], image["width"], image["channels"])) frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) results = model(frame, verbose=False) # includes NMS From c77c461ffb7b2eb5a3970066b9a86d529fb19e84 Mon Sep 17 00:00:00 2001 From: Enzo Le Van Date: Fri, 19 Jul 2024 14:14:17 +0200 Subject: [PATCH 08/18] Update examples/python-dataflow/README.md Co-authored-by: Philipp Oppermann --- examples/python-dataflow/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python-dataflow/README.md b/examples/python-dataflow/README.md index aa0c9be0..0ed2d2ac 100644 --- a/examples/python-dataflow/README.md +++ b/examples/python-dataflow/README.md @@ -16,8 +16,8 @@ information on how to start such a dataflow. ## Getting started After installing Rust, `dora-cli` and `Python >3.11`, you will need to **activate** (or create and **activate**) a -virtual -environment. Then, you will need to install the dependencies: +[Python virtual environment](https://docs.python.org/3/library/venv.html). +Then, you will need to install the dependencies: ```bash cd examples/python-dataflow From 8e30506bd3401f989084c4133cc79f0246233ff4 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jul 2024 06:58:35 +0200 Subject: [PATCH 09/18] Simplify example node --- examples/python-dataflow/dataflow.yml | 22 ++++++-- examples/python-dataflow/dataflow_yolo.yml | 36 ------------ node-hub/opencv-plot/README.md | 3 - node-hub/opencv-plot/main.py | 64 +++++++++++----------- node-hub/opencv-video-capture/README.md | 1 - node-hub/opencv-video-capture/main.py | 61 +++++++++++++++------ node-hub/ultralytics-yolo/main.py | 30 +++++++--- 7 files changed, 114 insertions(+), 103 deletions(-) delete mode 100644 examples/python-dataflow/dataflow_yolo.yml diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index e6a743ab..035242c9 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -4,20 +4,30 @@ nodes: path: opencv-video-capture inputs: tick: dora/timer/millis/16 - stop: plot/end outputs: - image - - text env: CAPTURE_PATH: 0 IMAGE_WIDTH: 640 IMAGE_HEIGHT: 480 + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: + source: camera/image + queue_size: 1 + outputs: + - bbox + env: + MODEL: yolov8n.pt + - id: plot build: pip install ../../node-hub/opencv-plot path: opencv-plot inputs: - image: camera/image - text: camera/text - outputs: - - end \ No newline at end of file + image: + source: camera/image + queue_size: 1 + bbox: object-detection/bbox diff --git a/examples/python-dataflow/dataflow_yolo.yml b/examples/python-dataflow/dataflow_yolo.yml deleted file mode 100644 index e5f8ca48..00000000 --- a/examples/python-dataflow/dataflow_yolo.yml +++ /dev/null @@ -1,36 +0,0 @@ -nodes: - - id: camera - build: pip install ../../node-hub/opencv-video-capture - path: opencv-video-capture - inputs: - tick: dora/timer/millis/16 - stop: plot/end - outputs: - - image - env: - CAPTURE_PATH: 0 - IMAGE_WIDTH: 640 - IMAGE_HEIGHT: 480 - - - id: object-detection - build: pip install ../../node-hub/ultralytics-yolo - path: ultralytics-yolo - inputs: - image: - source: camera/image - queue_size: 1 - outputs: - - bbox - env: - MODEL: yolov8n.pt - - - id: plot - build: pip install ../../node-hub/opencv-plot - path: opencv-plot - inputs: - image: - source: camera/image - queue_size: 1 - bbox: object-detection/bbox - outputs: - - end \ No newline at end of file diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md index 8f6cdef1..266d5685 100644 --- a/node-hub/opencv-plot/README.md +++ b/node-hub/opencv-plot/README.md @@ -13,9 +13,6 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o # bbox: Arrow array of bbox # text: Arrow array of size 1 containing the text to be plotted - outputs: - - end - env: PLOT_WIDTH: 640 # optional, default is image input width PLOT_HEIGHT: 480 # optional, default is image input height diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/main.py index 0648c3fd..656b68ff 100644 --- a/node-hub/opencv-plot/main.py +++ b/node-hub/opencv-plot/main.py @@ -7,7 +7,7 @@ import pyarrow as pa from dora import Node -RUNNER_CI = True if os.getenv("CI", False) == "true" else False +RUNNER_CI = True if os.getenv("CI") == "true" else False class Plot: @@ -71,14 +71,33 @@ def plot_frame(plot): def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. parser = argparse.ArgumentParser( - description="OpenCV Plotter: This node is used to plot text and bounding boxes on an image.") + description="OpenCV Plotter: This node is used to plot text and bounding boxes on an image." + ) - parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", - default="opencv-plot") - parser.add_argument("--plot-width", type=int, required=False, help="The width of the plot.", default=None) - parser.add_argument("--plot-height", type=int, required=False, help="The height of the plot.", default=None) + parser.add_argument( + "--name", + type=str, + required=False, + help="The name of the node in the dataflow.", + default="opencv-plot", + ) + parser.add_argument( + "--plot-width", + type=int, + required=False, + help="The width of the plot.", + default=None, + ) + parser.add_argument( + "--plot-height", + type=int, + required=False, + help="The height of the plot.", + default=None, + ) args = parser.parse_args() @@ -93,7 +112,9 @@ def main(): if isinstance(plot_height, str) and plot_height.isnumeric(): plot_height = int(plot_height) - node = Node(args.name) # provide the name to connect to the dataflow if dynamic node + node = Node( + args.name + ) # provide the name to connect to the dataflow if dynamic node plot = Plot() plot.width = plot_width @@ -113,18 +134,17 @@ def main(): "width": np.uint32(arrow_image["width"].as_py()), "height": np.uint32(arrow_image["height"].as_py()), "channels": np.uint8(arrow_image["channels"].as_py()), - "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + "data": arrow_image["data"].values.to_numpy().astype(np.uint8), } - plot.frame = np.reshape(image["data"], (image["height"], image["width"], image["channels"])) + plot.frame = np.reshape( + image["data"], (image["height"], image["width"], image["channels"]) + ) plot_frame(plot) if not RUNNER_CI: if cv2.waitKey(1) & 0xFF == ord("q"): break - else: - break # break the loop for CI - elif event_id == "bbox": arrow_bbox = event["value"][0] plot.bboxes = { @@ -132,31 +152,11 @@ def main(): "conf": arrow_bbox["conf"].values.to_numpy(), "names": arrow_bbox["names"].values.to_numpy(zero_copy_only=False), } - - plot_frame(plot) - if not RUNNER_CI: - if cv2.waitKey(1) & 0xFF == ord("q"): - break - else: - break # break the loop for CI elif event_id == "text": plot.text = event["value"][0].as_py() - - plot_frame(plot) - if not RUNNER_CI: - if cv2.waitKey(1) & 0xFF == ord("q"): - break - else: - break # break the loop for CI - elif event_type == "ERROR": raise Exception(event["error"]) - node.send_output( - "end", - pa.array([0], type=pa.uint8()) - ) - if __name__ == "__main__": main() diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md index 9aa2dbb8..216a47ed 100644 --- a/node-hub/opencv-video-capture/README.md +++ b/node-hub/opencv-video-capture/README.md @@ -10,7 +10,6 @@ This node is used to capture video from a camera using OpenCV. path: opencv-video-capture inputs: tick: dora/timer/millis/16 # try to capture at 60fps - # stop: some stop signal from another node outputs: - image: # the captured image diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py index 85a8e2c8..80255142 100644 --- a/node-hub/opencv-video-capture/main.py +++ b/node-hub/opencv-video-capture/main.py @@ -7,20 +7,45 @@ import pyarrow as pa from dora import Node +import time + +RUNNER_CI = True if os.getenv("CI") == "true" else False + def main(): # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. parser = argparse.ArgumentParser( - description="OpenCV Video Capture: This node is used to capture video from a camera.") - - parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", - default="opencv-video-capture") - parser.add_argument("--path", type=int, required=False, - help="The path of the device to capture (e.g. /dev/video1, or an index like 0, 1...", default=0) - parser.add_argument("--image-width", type=int, required=False, - help="The width of the image output. Default is the camera width.", default=None) - parser.add_argument("--image-height", type=int, required=False, - help="The height of the camera. Default is the camera height.", default=None) + description="OpenCV Video Capture: This node is used to capture video from a camera." + ) + + parser.add_argument( + "--name", + type=str, + required=False, + help="The name of the node in the dataflow.", + default="opencv-video-capture", + ) + parser.add_argument( + "--path", + type=int, + required=False, + help="The path of the device to capture (e.g. /dev/video1, or an index like 0, 1...", + default=0, + ) + parser.add_argument( + "--image-width", + type=int, + required=False, + help="The width of the image output. Default is the camera width.", + default=None, + ) + parser.add_argument( + "--image-height", + type=int, + required=False, + help="The height of the camera. Default is the camera height.", + default=None, + ) args = parser.parse_args() @@ -42,10 +67,16 @@ def main(): video_capture = cv2.VideoCapture(video_capture_path) node = Node(args.name) + start_time = time.time() pa.array([]) # initialize pyarrow array for event in node: + + # Run this eample in the CI for 20 seconds only. + if RUNNER_CI and time.time() - start_time > 20: + break + event_type = event["type"] if event_type == "INPUT": @@ -58,7 +89,7 @@ def main(): frame = np.zeros((480, 640, 3), dtype=np.uint8) cv2.putText( frame, - f'Error: no frame for camera at path {video_capture_path}.', + f"Error: no frame for camera at path {video_capture_path}.", (int(30), int(30)), cv2.FONT_HERSHEY_SIMPLEX, 0.50, @@ -75,14 +106,10 @@ def main(): "width": np.uint32(frame.shape[1]), "height": np.uint32(frame.shape[0]), "channels": np.uint8(frame.shape[2]), - "data": frame.ravel() + "data": frame.ravel(), } - node.send_output( - "image", - pa.array([image]), - event["metadata"] - ) + node.send_output("image", pa.array([image]), event["metadata"]) if event_id == "stop": video_capture.release() diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/main.py index fa3e48ba..abb3e048 100644 --- a/node-hub/ultralytics-yolo/main.py +++ b/node-hub/ultralytics-yolo/main.py @@ -7,15 +7,27 @@ import pyarrow as pa from dora import Node from ultralytics import YOLO + def main(): # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. parser = argparse.ArgumentParser( - description="UltraLytics YOLO: This node is used to perform object detection using the UltraLytics YOLO model.") - - parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", - default="ultralytics-yolo") - parser.add_argument("--model", type=str, required=False, - help="The name of the model file (e.g. yolov8n.pt).", default="yolov8n.pt") + description="UltraLytics YOLO: This node is used to perform object detection using the UltraLytics YOLO model." + ) + + parser.add_argument( + "--name", + type=str, + required=False, + help="The name of the node in the dataflow.", + default="ultralytics-yolo", + ) + parser.add_argument( + "--model", + type=str, + required=False, + help="The name of the model file (e.g. yolov8n.pt).", + default="yolov8n.pt", + ) args = parser.parse_args() @@ -38,10 +50,12 @@ def main(): "width": np.uint32(arrow_image["width"].as_py()), "height": np.uint32(arrow_image["height"].as_py()), "channels": np.uint8(arrow_image["channels"].as_py()), - "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + "data": arrow_image["data"].values.to_numpy().astype(np.uint8), } - frame = image["data"].reshape((image["height"], image["width"], image["channels"])) + frame = image["data"].reshape( + (image["height"], image["width"], image["channels"]) + ) frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) results = model(frame, verbose=False) # includes NMS From 8878807acd639e1aa3c6a22070b7f40fe606e60c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jul 2024 07:45:07 +0200 Subject: [PATCH 10/18] Adding disk freeing on CLI Test fix typo --- .github/workflows/ci.yml | 18 +++++++++++++++++- node-hub/opencv-video-capture/main.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faee5960..d3cba5b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,6 +264,22 @@ jobs: - uses: actions/checkout@v3 - uses: r7kamura/rust-problem-matchers@v1.1.0 - run: cargo --version --verbose + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + if: runner.os == 'Linux' + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: false - uses: Swatinem/rust-cache@v2 with: cache-provider: buildjet @@ -347,7 +363,7 @@ jobs: sleep 10 dora stop --name ci-c-test --grace-duration 5s dora destroy - + - name: "Test CLI (C++)" timeout-minutes: 30 # fail-fast by using bash shell explictly diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py index 80255142..892cc85b 100644 --- a/node-hub/opencv-video-capture/main.py +++ b/node-hub/opencv-video-capture/main.py @@ -73,7 +73,7 @@ def main(): for event in node: - # Run this eample in the CI for 20 seconds only. + # Run this example in the CI for 20 seconds only. if RUNNER_CI and time.time() - start_time > 20: break From e52834b5c4c3e5d5c75c782990798e07b510dca7 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jul 2024 15:42:47 +0200 Subject: [PATCH 11/18] Reformat folder so that they can be used in edit mode as well as use encoding to support multiple encoding --- examples/python-dataflow/dataflow.yml | 8 ++--- node-hub/opencv-plot/README.md | 35 +++++++++---------- .../opencv-plot/{ => opencv_plot}/main.py | 13 +++++-- node-hub/opencv-plot/pyproject.toml | 8 ++--- node-hub/opencv-video-capture/README.md | 34 +++++++++--------- .../{ => opencv_video_capture}/main.py | 13 ++++--- node-hub/opencv-video-capture/pyproject.toml | 8 ++--- node-hub/ultralytics-yolo/README.md | 8 ++--- node-hub/ultralytics-yolo/pyproject.toml | 8 ++--- .../{ => ultralytics_yolo}/main.py | 17 +++++++-- 10 files changed, 82 insertions(+), 70 deletions(-) rename node-hub/opencv-plot/{ => opencv_plot}/main.py (91%) rename node-hub/opencv-video-capture/{ => opencv_video_capture}/main.py (95%) rename node-hub/ultralytics-yolo/{ => ultralytics_yolo}/main.py (82%) diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 035242c9..44cd8409 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -1,9 +1,9 @@ nodes: - id: camera - build: pip install ../../node-hub/opencv-video-capture + build: pip install -e ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: - tick: dora/timer/millis/16 + tick: dora/timer/millis/20 outputs: - image env: @@ -12,7 +12,7 @@ nodes: IMAGE_HEIGHT: 480 - id: object-detection - build: pip install ../../node-hub/ultralytics-yolo + build: pip install -e ../../node-hub/ultralytics-yolo path: ultralytics-yolo inputs: image: @@ -24,7 +24,7 @@ nodes: MODEL: yolov8n.pt - id: plot - build: pip install ../../node-hub/opencv-plot + build: pip install -e ../../node-hub/opencv-plot path: opencv-plot inputs: image: diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md index 266d5685..ee77c86c 100644 --- a/node-hub/opencv-plot/README.md +++ b/node-hub/opencv-plot/README.md @@ -5,17 +5,17 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o # YAML ```yaml - - id: opencv-plot - build: pip install ../../node-hub/opencv-plot - path: opencv-plot - inputs: - # image: Arrow array of size 1 containing the base image - # bbox: Arrow array of bbox - # text: Arrow array of size 1 containing the text to be plotted - - env: - PLOT_WIDTH: 640 # optional, default is image input width - PLOT_HEIGHT: 480 # optional, default is image input height +- id: opencv-plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + # image: Arrow array of size 1 containing the base image + # bbox: Arrow array of bbox + # text: Arrow array of size 1 containing the text to be plotted + + env: + PLOT_WIDTH: 640 # optional, default is image input width + PLOT_HEIGHT: 480 # optional, default is image input height ``` # Inputs @@ -26,19 +26,18 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o image: { "width": np.uint32, "height": np.uint32, - "channels": np.uint8, + "encoding": bytes, "data": np.array # flattened image data } encoded_image = pa.array([image]) decoded_image = { - "width": np.uint32(encoded_image[0]["width"].as_py()), - "height": np.uint32(encoded_image[0]["height"].as_py()), - "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "width": np.uint32(encoded_image[0]["width"]), + "height": np.uint32(encoded_image[0]["height"]), + "encoding": encoded_image[0]["encoding"].as_py(), "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) -} - +} ``` - `bbox`: an arrow array containing the bounding boxes, confidence scores, and class names of the detected objects @@ -68,7 +67,7 @@ text: str encoded_text = pa.array([text]) decoded_text = encoded_text[0].as_py() -``` +``` ## License diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/opencv_plot/main.py similarity index 91% rename from node-hub/opencv-plot/main.py rename to node-hub/opencv-plot/opencv_plot/main.py index 656b68ff..6b89026d 100644 --- a/node-hub/opencv-plot/main.py +++ b/node-hub/opencv-plot/opencv_plot/main.py @@ -130,11 +130,20 @@ def main(): if event_id == "image": arrow_image = event["value"][0] + + encoding = arrow_image["encoding"].as_py() + if encoding == "bgr8": + channels = 3 + storage_type = np.uint8 + else: + raise Exception(f"Unsupported image encoding: {encoding}") + image = { "width": np.uint32(arrow_image["width"].as_py()), "height": np.uint32(arrow_image["height"].as_py()), - "channels": np.uint8(arrow_image["channels"].as_py()), - "data": arrow_image["data"].values.to_numpy().astype(np.uint8), + "encoding": encoding, + "channels": channels, + "data": arrow_image["data"].values.to_numpy().astype(storage_type), } plot.frame = np.reshape( diff --git a/node-hub/opencv-plot/pyproject.toml b/node-hub/opencv-plot/pyproject.toml index a1f01c45..a6791fe3 100644 --- a/node-hub/opencv-plot/pyproject.toml +++ b/node-hub/opencv-plot/pyproject.toml @@ -3,14 +3,12 @@ name = "opencv-plot" version = "0.1" authors = [ "Haixuan Xavier Tao ", - "Enzo Le Van " + "Enzo Le Van ", ] description = "Dora Node for plotting text and bbox on image with OpenCV" readme = "README.md" -packages = [ - { include = "main.py", to = "opencv_plot" } -] +packages = [{ include = "opencv_plot" }] [tool.poetry.dependencies] dora-rs = "0.3.5" @@ -22,4 +20,4 @@ opencv-plot = "opencv_plot.main:main" [build-system] requires = ["poetry-core>=1.8.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md index 216a47ed..923b9400 100644 --- a/node-hub/opencv-video-capture/README.md +++ b/node-hub/opencv-video-capture/README.md @@ -5,19 +5,19 @@ This node is used to capture video from a camera using OpenCV. # YAML ```yaml - - id: opencv-video-capture - build: pip install ../../node-hub/opencv-video-capture - path: opencv-video-capture - inputs: - tick: dora/timer/millis/16 # try to capture at 60fps - outputs: - - image: # the captured image - - env: - PATH: 0 # optional, default is 0 - - IMAGE_WIDTH: 640 # optional, default is video capture width - IMAGE_HEIGHT: 480 # optional, default is video capture height +- id: opencv-video-capture + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: dora/timer/millis/16 # try to capture at 60fps + outputs: + - image: # the captured image + + env: + PATH: 0 # optional, default is 0 + + IMAGE_WIDTH: 640 # optional, default is video capture width + IMAGE_HEIGHT: 480 # optional, default is video capture height ``` # Inputs @@ -33,16 +33,16 @@ This node is used to capture video from a camera using OpenCV. image: { "width": np.uint32, "height": np.uint32, - "channels": np.uint8, + "encoding": str, "data": np.array # flattened image data } encoded_image = pa.array([image]) decoded_image = { - "width": np.uint32(encoded_image[0]["width"].as_py()), - "height": np.uint32(encoded_image[0]["height"].as_py()), - "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "width": np.uint32(encoded_image[0]["width"]), + "height": np.uint32(encoded_image[0]["height"]), + "encoding": encoded_image[0]["encoding"].as_py(), "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) } ``` diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/opencv_video_capture/main.py similarity index 95% rename from node-hub/opencv-video-capture/main.py rename to node-hub/opencv-video-capture/opencv_video_capture/main.py index 892cc85b..54e3886a 100644 --- a/node-hub/opencv-video-capture/main.py +++ b/node-hub/opencv-video-capture/opencv_video_capture/main.py @@ -54,18 +54,21 @@ def main(): if isinstance(video_capture_path, str) and video_capture_path.isnumeric(): video_capture_path = int(video_capture_path) + video_capture = cv2.VideoCapture(video_capture_path) + image_width = os.getenv("IMAGE_WIDTH", args.image_width) - image_height = os.getenv("IMAGE_HEIGHT", args.image_height) if image_width is not None: if isinstance(image_width, str) and image_width.isnumeric(): image_width = int(image_width) + video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, image_width) + image_height = os.getenv("IMAGE_HEIGHT", args.image_height) if image_height is not None: if isinstance(image_height, str) and image_height.isnumeric(): image_height = int(image_height) + video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, image_height) - video_capture = cv2.VideoCapture(video_capture_path) node = Node(args.name) start_time = time.time() @@ -105,16 +108,12 @@ def main(): image = { "width": np.uint32(frame.shape[1]), "height": np.uint32(frame.shape[0]), - "channels": np.uint8(frame.shape[2]), + "encoding": "bgr8", "data": frame.ravel(), } node.send_output("image", pa.array([image]), event["metadata"]) - if event_id == "stop": - video_capture.release() - break - elif event_type == "ERROR": raise Exception(event["error"]) diff --git a/node-hub/opencv-video-capture/pyproject.toml b/node-hub/opencv-video-capture/pyproject.toml index 00100acf..5d686ebd 100644 --- a/node-hub/opencv-video-capture/pyproject.toml +++ b/node-hub/opencv-video-capture/pyproject.toml @@ -3,14 +3,12 @@ name = "opencv-video-capture" version = "0.1" authors = [ "Haixuan Xavier Tao ", - "Enzo Le Van " + "Enzo Le Van ", ] description = "Dora Node for capturing video with OpenCV" readme = "README.md" -packages = [ - { include = "main.py", to = "opencv_video_capture" } -] +packages = [{ include = "opencv_video_capture" }] [tool.poetry.dependencies] dora-rs = "0.3.5" @@ -22,4 +20,4 @@ opencv-video-capture = "opencv_video_capture.main:main" [build-system] requires = ["poetry-core>=1.8.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/node-hub/ultralytics-yolo/README.md b/node-hub/ultralytics-yolo/README.md index 8a7bc8f2..fae47540 100644 --- a/node-hub/ultralytics-yolo/README.md +++ b/node-hub/ultralytics-yolo/README.md @@ -25,16 +25,16 @@ This node is used to detect objects in images using YOLOv8. image: { "width": np.uint32, "height": np.uint32, - "channels": np.uint8, + "encoding": str, "data": np.array # flattened image data } encoded_image = pa.array([image]) decoded_image = { - "width": np.uint32(encoded_image[0]["width"].as_py()), - "height": np.uint32(encoded_image[0]["height"].as_py()), - "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "width": np.uint32(encoded_image[0]["width"]), + "height": np.uint32(encoded_image[0]["height"]), + "encoding": encoded_image[0]["encoding"].as_py(), "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) } diff --git a/node-hub/ultralytics-yolo/pyproject.toml b/node-hub/ultralytics-yolo/pyproject.toml index 0d7feacf..2665168a 100644 --- a/node-hub/ultralytics-yolo/pyproject.toml +++ b/node-hub/ultralytics-yolo/pyproject.toml @@ -3,14 +3,12 @@ name = "ultralytics-yolo" version = "0.1" authors = [ "Haixuan Xavier Tao ", - "Enzo Le Van " + "Enzo Le Van ", ] description = "Dora Node for object detection with Ultralytics YOLOv8" readme = "README.md" -packages = [ - { include = "main.py", to = "ultralytics_yolo" } -] +packages = [{ include = "ultralytics_yolo" }] [tool.poetry.dependencies] dora-rs = "0.3.5" @@ -22,4 +20,4 @@ ultralytics-yolo = "ultralytics_yolo.main:main" [build-system] requires = ["poetry-core>=1.8.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py similarity index 82% rename from node-hub/ultralytics-yolo/main.py rename to node-hub/ultralytics-yolo/ultralytics_yolo/main.py index abb3e048..023742c8 100644 --- a/node-hub/ultralytics-yolo/main.py +++ b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py @@ -46,18 +46,29 @@ def main(): if event_id == "image": arrow_image = event["value"][0] + encoding = arrow_image["encoding"].as_py() + + if encoding == "bgr8": + channels = 3 + storage_type = np.uint8 + else: + raise Exception(f"Unsupported image encoding: {encoding}") + image = { "width": np.uint32(arrow_image["width"].as_py()), "height": np.uint32(arrow_image["height"].as_py()), - "channels": np.uint8(arrow_image["channels"].as_py()), - "data": arrow_image["data"].values.to_numpy().astype(np.uint8), + "encoding": encoding, + "channels": channels, + "data": arrow_image["data"].values.to_numpy().astype(storage_type), } frame = image["data"].reshape( (image["height"], image["width"], image["channels"]) ) - frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + if encoding == "bgr8": + frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + results = model(frame, verbose=False) # includes NMS bboxes = np.array(results[0].boxes.xyxy.cpu()) From f89b171bbcf164770066e3d7061c49aac8900820 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jul 2024 15:55:47 +0200 Subject: [PATCH 12/18] remove editable mode as it is not supported on older pip version --- examples/python-dataflow/dataflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 44cd8409..defc78ad 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -1,6 +1,6 @@ nodes: - id: camera - build: pip install -e ../../node-hub/opencv-video-capture + build: pip install ../../node-hub/opencv-video-capture path: opencv-video-capture inputs: tick: dora/timer/millis/20 @@ -12,7 +12,7 @@ nodes: IMAGE_HEIGHT: 480 - id: object-detection - build: pip install -e ../../node-hub/ultralytics-yolo + build: pip install ../../node-hub/ultralytics-yolo path: ultralytics-yolo inputs: image: @@ -24,7 +24,7 @@ nodes: MODEL: yolov8n.pt - id: plot - build: pip install -e ../../node-hub/opencv-plot + build: pip install ../../node-hub/opencv-plot path: opencv-plot inputs: image: From 00d3732a902ec55c4584ec23f602e7962af017eb Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Fri, 19 Jul 2024 17:12:23 +0200 Subject: [PATCH 13/18] remove unused input and output from dynamic node --- examples/python-dataflow/dataflow_dynamic.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 4253e2d8..991bd2b3 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -4,7 +4,6 @@ nodes: path: opencv-video-capture inputs: tick: dora/timer/millis/16 - stop: plot/end outputs: - image env: @@ -17,5 +16,3 @@ nodes: path: dynamic inputs: image: camera/image - outputs: - - end \ No newline at end of file From bc68de3bbd900fa90001d446e54ac0087b3ba7ef Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 21 Jul 2024 08:00:13 +0200 Subject: [PATCH 14/18] Change `MetadataParameters` into a `BTreeMap` to allow user defined metadata as well as enable more flexibility in managing metadata --- apis/python/operator/src/lib.rs | 86 +++++++++++++++++---------------- apis/rust/node/src/lib.rs | 2 +- apis/rust/node/src/node/mod.rs | 6 +-- libraries/message/src/lib.rs | 31 ++++++------ 4 files changed, 64 insertions(+), 61 deletions(-) diff --git a/apis/python/operator/src/lib.rs b/apis/python/operator/src/lib.rs index 1929bc36..4ac5080c 100644 --- a/apis/python/operator/src/lib.rs +++ b/apis/python/operator/src/lib.rs @@ -1,20 +1,19 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, sync::{Arc, Mutex}, }; use arrow::pyarrow::ToPyArrow; use dora_node_api::{ merged::{MergeExternalSend, MergedEvent}, - DoraNode, Event, EventStream, Metadata, MetadataParameters, + DoraNode, Event, EventStream, Metadata, MetadataParameters, Parameter, }; use eyre::{Context, Result}; use futures::{Stream, StreamExt}; use futures_concurrency::stream::Merge as _; use pyo3::{ prelude::*, - pybacked::PyBackedStr, - types::{IntoPyDict, PyDict}, + types::{IntoPyDict, PyBool, PyDict, PyInt, PyString}, }; /// Dora Event @@ -94,7 +93,7 @@ impl PyEvent { if let Some(value) = self.value(py)? { pydict.insert("value", value); } - if let Some(metadata) = Self::metadata(event, py) { + if let Some(metadata) = Self::metadata(event, py)? { pydict.insert("metadata", metadata); } if let Some(error) = Self::error(event) { @@ -143,10 +142,14 @@ impl PyEvent { } } - fn metadata(event: &Event, py: Python<'_>) -> Option { + fn metadata(event: &Event, py: Python<'_>) -> Result> { match event { - Event::Input { metadata, .. } => Some(metadata_to_pydict(metadata, py).to_object(py)), - _ => None, + Event::Input { metadata, .. } => Ok(Some( + metadata_to_pydict(metadata, py) + .context("Issue deserializing metadata")? + .to_object(py), + )), + _ => Ok(None), } } @@ -159,44 +162,45 @@ impl PyEvent { } pub fn pydict_to_metadata(dict: Option>) -> Result { - let mut default_metadata = MetadataParameters::default(); - if let Some(metadata) = dict { - for (key, value) in metadata.iter() { - match key - .extract::() - .context("Parsing metadata keys")? - .as_ref() - { - "watermark" => { - default_metadata.watermark = - value.extract().context("parsing watermark failed")?; - } - "deadline" => { - default_metadata.deadline = - value.extract().context("parsing deadline failed")?; - } - "open_telemetry_context" => { - let otel_context: PyBackedStr = value - .extract() - .context("parsing open telemetry context failed")?; - default_metadata.open_telemetry_context = otel_context.to_string(); - } - _ => (), - } + let mut parameters = BTreeMap::default(); + if let Some(pymetadata) = dict { + for (key, value) in pymetadata.iter() { + let key = key.extract::().context("Parsing metadata keys")?; + if value.is_exact_instance_of::() { + parameters.insert(key, Parameter::Bool(value.extract()?)) + } else if value.is_instance_of::() { + parameters.insert(key, Parameter::Integer(value.extract::()?)) + } else if value.is_instance_of::() { + parameters.insert(key, Parameter::String(value.extract()?)) + } else { + println!("could not convert type {value}"); + parameters.insert(key, Parameter::String(value.str()?.to_string())) + }; } } - Ok(default_metadata) + Ok(parameters) } -pub fn metadata_to_pydict<'a>(metadata: &'a Metadata, py: Python<'a>) -> pyo3::Bound<'a, PyDict> { +pub fn metadata_to_pydict<'a>( + metadata: &'a Metadata, + py: Python<'a>, +) -> Result> { let dict = PyDict::new_bound(py); - dict.set_item( - "open_telemetry_context", - &metadata.parameters.open_telemetry_context, - ) - .wrap_err("could not make metadata a python dictionary item") - .unwrap(); - dict + for (k, v) in metadata.parameters.iter() { + match v { + Parameter::Bool(bool) => dict + .set_item(k, bool) + .context(format!("Could not insert metadata into python dictionary"))?, + Parameter::Integer(int) => dict + .set_item(k, int) + .context(format!("Could not insert metadata into python dictionary"))?, + Parameter::String(s) => dict + .set_item(k, s) + .context(format!("Could not insert metadata into python dictionary"))?, + } + } + + Ok(dict) } #[cfg(test)] diff --git a/apis/rust/node/src/lib.rs b/apis/rust/node/src/lib.rs index d2ce63dd..7c61559e 100644 --- a/apis/rust/node/src/lib.rs +++ b/apis/rust/node/src/lib.rs @@ -16,7 +16,7 @@ pub use arrow; pub use dora_arrow_convert::*; pub use dora_core; -pub use dora_core::message::{uhlc, Metadata, MetadataParameters}; +pub use dora_core::message::{uhlc, Metadata, MetadataParameters, Parameter}; pub use event_stream::{merged, Event, EventStream, MappedInputData, RawData}; pub use flume::Receiver; pub use node::{arrow_utils, DataSample, DoraNode, ZERO_COPY_THRESHOLD}; diff --git a/apis/rust/node/src/node/mod.rs b/apis/rust/node/src/node/mod.rs index 9eb4b18e..8951deb5 100644 --- a/apis/rust/node/src/node/mod.rs +++ b/apis/rust/node/src/node/mod.rs @@ -250,11 +250,7 @@ impl DoraNode { if !self.node_config.outputs.contains(&output_id) { eyre::bail!("unknown output"); } - let metadata = Metadata::from_parameters( - self.clock.new_timestamp(), - type_info, - parameters.into_owned(), - ); + let metadata = Metadata::from_parameters(self.clock.new_timestamp(), type_info, parameters); let (data, shmem) = match sample { Some(sample) => sample.finalize(), diff --git a/libraries/message/src/lib.rs b/libraries/message/src/lib.rs index 40c18f99..fa69c2fb 100644 --- a/libraries/message/src/lib.rs +++ b/libraries/message/src/lib.rs @@ -3,12 +3,16 @@ #![allow(clippy::missing_safety_doc)] +use std::collections::BTreeMap; + use arrow_data::ArrayData; use arrow_schema::DataType; use eyre::Context; use serde::{Deserialize, Serialize}; pub use uhlc; +pub type MetadataParameters = BTreeMap; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Metadata { metadata_version: u16, @@ -105,20 +109,11 @@ pub struct BufferOffset { pub len: usize, } -#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)] -pub struct MetadataParameters { - pub watermark: u64, - pub deadline: u64, - pub open_telemetry_context: String, -} - -impl MetadataParameters { - pub fn into_owned(self) -> MetadataParameters { - MetadataParameters { - open_telemetry_context: self.open_telemetry_context, - ..self - } - } +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub enum Parameter { + Bool(bool), + Integer(i64), + String(String), } impl Metadata { @@ -142,4 +137,12 @@ impl Metadata { pub fn timestamp(&self) -> uhlc::Timestamp { self.timestamp } + + pub fn open_telemetry_context(&self) -> String { + if let Some(Parameter::String(otel)) = self.parameters.get("open_telemetry_context") { + otel.to_string() + } else { + "".to_string() + } + } } From 7bcb132075d703d697c621895c509c739ab36166 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 21 Jul 2024 08:01:44 +0200 Subject: [PATCH 15/18] Retrieve `open_telemetry_context` from metadata parameters. --- binaries/daemon/src/lib.rs | 21 +++++++++------- binaries/runtime/src/operator/python.rs | 23 ++++++++++++------ binaries/runtime/src/operator/shared_lib.rs | 27 ++++++++++++--------- node-hub/dora-record/src/main.rs | 9 +++++-- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs index de799ad6..d8e19e3d 100644 --- a/binaries/daemon/src/lib.rs +++ b/binaries/daemon/src/lib.rs @@ -8,7 +8,7 @@ use dora_core::daemon_messages::{ }; use dora_core::descriptor::runtime_node_inputs; use dora_core::message::uhlc::{self, HLC}; -use dora_core::message::{ArrowTypeInfo, Metadata, MetadataParameters}; +use dora_core::message::{ArrowTypeInfo, Metadata}; use dora_core::topics::LOCALHOST; use dora_core::topics::{ DataflowDaemonResult, DataflowResult, NodeError, NodeErrorCause, NodeExitStatus, @@ -23,6 +23,7 @@ use dora_core::{ descriptor::{CoreNodeKind, Descriptor, ResolvedNode}, }; +use dora_node_api::Parameter; use eyre::{bail, eyre, Context, ContextCompat, Result}; use futures::{future, stream, FutureExt, TryFutureExt}; use futures_concurrency::stream::Merge; @@ -1543,17 +1544,19 @@ impl RunningDataflow { let span = tracing::span!(tracing::Level::TRACE, "tick"); let _ = span.enter(); + let mut parameters = BTreeMap::new(); + parameters.insert( + "open_telemetry_context".to_string(), + #[cfg(feature = "telemetry")] + Parameter::String(serialize_context(&span.context())), + #[cfg(not(feature = "telemetry"))] + Parameter::String("".into()), + ); + let metadata = dora_core::message::Metadata::from_parameters( hlc.new_timestamp(), ArrowTypeInfo::empty(), - MetadataParameters { - watermark: 0, - deadline: 0, - #[cfg(feature = "telemetry")] - open_telemetry_context: serialize_context(&span.context()), - #[cfg(not(feature = "telemetry"))] - open_telemetry_context: "".into(), - }, + parameters, ); let event = Timestamped { diff --git a/binaries/runtime/src/operator/python.rs b/binaries/runtime/src/operator/python.rs index fd436f1a..21e00ec6 100644 --- a/binaries/runtime/src/operator/python.rs +++ b/binaries/runtime/src/operator/python.rs @@ -6,7 +6,7 @@ use dora_core::{ descriptor::{source_is_url, Descriptor, PythonSource}, }; use dora_download::download_file; -use dora_node_api::{merged::MergedEvent, Event}; +use dora_node_api::{merged::MergedEvent, Event, Parameter}; use dora_operator_api_python::PyEvent; use dora_operator_api_types::DoraStatus; use eyre::{bail, eyre, Context, Result}; @@ -201,11 +201,15 @@ pub fn run( use tracing_opentelemetry::OpenTelemetrySpanExt; span.record("input_id", input_id.as_str()); - let cx = deserialize_context(&metadata.parameters.open_telemetry_context); + let otel = metadata.open_telemetry_context(); + let cx = deserialize_context(&otel); span.set_parent(cx); let cx = span.context(); let string_cx = serialize_context(&cx); - metadata.parameters.open_telemetry_context = string_cx; + metadata.parameters.insert( + "open_telemetry_context".to_string(), + Parameter::String(string_cx), + ); } let py_event = PyEvent { @@ -317,17 +321,22 @@ mod callback_impl { metadata: Option>, py: Python, ) -> Result<()> { - let parameters = pydict_to_metadata(metadata) - .wrap_err("failed to parse metadata")? - .into_owned(); + let parameters = pydict_to_metadata(metadata).wrap_err("failed to parse metadata")?; let span = span!( tracing::Level::TRACE, "send_output", output_id = field::Empty ); span.record("output_id", output); + let otel = if let Some(dora_node_api::Parameter::String(otel)) = + parameters.get("open_telemetry_context") + { + otel.to_string() + } else { + "".to_string() + }; - let cx = deserialize_context(¶meters.open_telemetry_context); + let cx = deserialize_context(&otel); span.set_parent(cx); let _ = span.enter(); diff --git a/binaries/runtime/src/operator/shared_lib.rs b/binaries/runtime/src/operator/shared_lib.rs index 984a760b..70fccff4 100644 --- a/binaries/runtime/src/operator/shared_lib.rs +++ b/binaries/runtime/src/operator/shared_lib.rs @@ -8,7 +8,7 @@ use dora_core::{ use dora_download::download_file; use dora_node_api::{ arrow_utils::{copy_array_into_sample, required_data_size}, - Event, MetadataParameters, + Event, Parameter, }; use dora_operator_api_types::{ safer_ffi::closure::ArcDynFn1, DoraDropOperator, DoraInitOperator, DoraInitResult, DoraOnEvent, @@ -17,6 +17,7 @@ use dora_operator_api_types::{ use eyre::{bail, eyre, Context, Result}; use libloading::Symbol; use std::{ + collections::BTreeMap, ffi::c_void, panic::{catch_unwind, AssertUnwindSafe}, path::Path, @@ -119,10 +120,11 @@ impl<'lib> SharedLibraryOperator<'lib> { open_telemetry_context, }, } = output; - let parameters = MetadataParameters { - open_telemetry_context: open_telemetry_context.into(), - ..Default::default() - }; + let mut parameters = BTreeMap::new(); + parameters.insert( + "open_telemetry_context".to_string(), + Parameter::String(open_telemetry_context.to_string()), + ); let arrow_array = match unsafe { arrow::ffi::from_ffi(data_array, &schema) } { Ok(a) => a, @@ -173,11 +175,15 @@ impl<'lib> SharedLibraryOperator<'lib> { use tracing_opentelemetry::OpenTelemetrySpanExt; span.record("input_id", input_id.as_str()); - let cx = deserialize_context(&metadata.parameters.open_telemetry_context); + let otel = metadata.open_telemetry_context(); + let cx = deserialize_context(&otel); span.set_parent(cx); let cx = span.context(); let string_cx = serialize_context(&cx); - metadata.parameters.open_telemetry_context = string_cx; + metadata.parameters.insert( + "open_telemetry_context".to_string(), + Parameter::String(string_cx), + ); } let mut operator_event = match event { @@ -193,16 +199,13 @@ impl<'lib> SharedLibraryOperator<'lib> { data, } => { let (data_array, schema) = arrow::ffi::to_ffi(&data.to_data())?; - + let otel = metadata.open_telemetry_context(); let operator_input = dora_operator_api_types::Input { id: String::from(input_id).into(), data_array: Some(data_array), schema, metadata: Metadata { - open_telemetry_context: metadata - .parameters - .open_telemetry_context - .into(), + open_telemetry_context: otel.into(), }, }; dora_operator_api_types::RawEvent { diff --git a/node-hub/dora-record/src/main.rs b/node-hub/dora-record/src/main.rs index 3d9268fe..0737583f 100644 --- a/node-hub/dora-record/src/main.rs +++ b/node-hub/dora-record/src/main.rs @@ -100,7 +100,12 @@ async fn main() -> eyre::Result<()> { None => {} Some(tx) => drop(tx), }, - _ => {} + Event::Error(err) => { + println!("Error: {}", err); + } + event => { + println!("Event: {event:#?}") + } } } @@ -137,7 +142,7 @@ async fn write_event( let timestamp_utc = TimestampMillisecondArray::from(vec![dt.timestamp_millis()]); let timestamp_utc = make_array(timestamp_utc.into()); - let string_otel_context = metadata.parameters.open_telemetry_context.to_string(); + let string_otel_context = metadata.open_telemetry_context(); let otel_context = deserialize_to_hashmap(&string_otel_context); let traceparent = otel_context.get("traceparent"); let trace_id = match traceparent { From b8d2e5560f1ff9b5ba6382e68ef1468705ab2524 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 21 Jul 2024 08:07:53 +0200 Subject: [PATCH 16/18] Fix `pylint` warning --- node-hub/opencv-plot/opencv_plot/main.py | 8 ++++---- .../opencv-video-capture/opencv_video_capture/main.py | 4 ++-- node-hub/ultralytics-yolo/ultralytics_yolo/main.py | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/node-hub/opencv-plot/opencv_plot/main.py b/node-hub/opencv-plot/opencv_plot/main.py index 6b89026d..bd43469f 100644 --- a/node-hub/opencv-plot/opencv_plot/main.py +++ b/node-hub/opencv-plot/opencv_plot/main.py @@ -1,6 +1,6 @@ import os import argparse -import cv2 +from cv2 import cv2 # Importing cv2 this way remove error warnings`` import numpy as np import pyarrow as pa @@ -13,7 +13,7 @@ RUNNER_CI = True if os.getenv("CI") == "true" else False class Plot: frame: np.array = np.array([]) - bboxes: {} = { + bboxes: dict = { "bbox": np.array([]), "conf": np.array([]), "names": np.array([]), @@ -136,7 +136,7 @@ def main(): channels = 3 storage_type = np.uint8 else: - raise Exception(f"Unsupported image encoding: {encoding}") + raise RuntimeError(f"Unsupported image encoding: {encoding}") image = { "width": np.uint32(arrow_image["width"].as_py()), @@ -164,7 +164,7 @@ def main(): elif event_id == "text": plot.text = event["value"][0].as_py() elif event_type == "ERROR": - raise Exception(event["error"]) + raise RuntimeError(event["error"]) if __name__ == "__main__": diff --git a/node-hub/opencv-video-capture/opencv_video_capture/main.py b/node-hub/opencv-video-capture/opencv_video_capture/main.py index 54e3886a..d74408e5 100644 --- a/node-hub/opencv-video-capture/opencv_video_capture/main.py +++ b/node-hub/opencv-video-capture/opencv_video_capture/main.py @@ -1,6 +1,6 @@ import os import argparse -import cv2 +from cv2 import cv2 # Importing cv2 this way remove error warnings`` import numpy as np import pyarrow as pa @@ -115,7 +115,7 @@ def main(): node.send_output("image", pa.array([image]), event["metadata"]) elif event_type == "ERROR": - raise Exception(event["error"]) + raise RuntimeError(event["error"]) if __name__ == "__main__": diff --git a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py index 023742c8..98b9a981 100644 --- a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py +++ b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py @@ -52,7 +52,7 @@ def main(): channels = 3 storage_type = np.uint8 else: - raise Exception(f"Unsupported image encoding: {encoding}") + raise RuntimeError(f"Unsupported image encoding: {encoding}") image = { "width": np.uint32(arrow_image["width"].as_py()), @@ -68,6 +68,8 @@ def main(): if encoding == "bgr8": frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + else: + raise RuntimeError(f"Unsupported image encoding: {encoding}") results = model(frame, verbose=False) # includes NMS @@ -90,7 +92,7 @@ def main(): ) elif event_type == "ERROR": - raise Exception(event["error"]) + raise RuntimeError(event["error"]) if __name__ == "__main__": From 2927e84d3d49ff3caa17674380c20078d134cf93 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Sun, 21 Jul 2024 08:08:30 +0200 Subject: [PATCH 17/18] Use dora metadata to reduce complexity as well as enable zero-copy image transfering --- node-hub/opencv-plot/opencv_plot/main.py | 25 +++++++++---------- .../opencv_video_capture/main.py | 16 ++++++------ .../ultralytics-yolo/ultralytics_yolo/main.py | 22 +++++++--------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/node-hub/opencv-plot/opencv_plot/main.py b/node-hub/opencv-plot/opencv_plot/main.py index bd43469f..7d8af16e 100644 --- a/node-hub/opencv-plot/opencv_plot/main.py +++ b/node-hub/opencv-plot/opencv_plot/main.py @@ -1,6 +1,6 @@ import os import argparse -from cv2 import cv2 # Importing cv2 this way remove error warnings`` +import cv2 import numpy as np import pyarrow as pa @@ -129,25 +129,24 @@ def main(): event_id = event["id"] if event_id == "image": - arrow_image = event["value"][0] + storage = event["value"] + + metadata = event["metadata"] + encoding = metadata["encoding"] + width = metadata["width"] + height = metadata["height"] - encoding = arrow_image["encoding"].as_py() if encoding == "bgr8": channels = 3 storage_type = np.uint8 else: raise RuntimeError(f"Unsupported image encoding: {encoding}") - image = { - "width": np.uint32(arrow_image["width"].as_py()), - "height": np.uint32(arrow_image["height"].as_py()), - "encoding": encoding, - "channels": channels, - "data": arrow_image["data"].values.to_numpy().astype(storage_type), - } - - plot.frame = np.reshape( - image["data"], (image["height"], image["width"], image["channels"]) + plot.frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) + .copy() # Copy So that we can add annotation on the image ) plot_frame(plot) diff --git a/node-hub/opencv-video-capture/opencv_video_capture/main.py b/node-hub/opencv-video-capture/opencv_video_capture/main.py index d74408e5..d9dee5e1 100644 --- a/node-hub/opencv-video-capture/opencv_video_capture/main.py +++ b/node-hub/opencv-video-capture/opencv_video_capture/main.py @@ -1,6 +1,6 @@ import os import argparse -from cv2 import cv2 # Importing cv2 this way remove error warnings`` +import cv2 import numpy as np import pyarrow as pa @@ -105,14 +105,14 @@ def main(): if image_width is not None and image_height is not None: frame = cv2.resize(frame, (image_width, image_height)) - image = { - "width": np.uint32(frame.shape[1]), - "height": np.uint32(frame.shape[0]), - "encoding": "bgr8", - "data": frame.ravel(), - } + storage = pa.array(frame.ravel()) - node.send_output("image", pa.array([image]), event["metadata"]) + metadata = event["metadata"] + metadata["width"] = int(frame.shape[1]) + metadata["height"] = int(frame.shape[0]) + metadata["encoding"] = "bgr8" + + node.send_output("image", storage, metadata) elif event_type == "ERROR": raise RuntimeError(event["error"]) diff --git a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py index 98b9a981..23ca85b0 100644 --- a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py +++ b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py @@ -45,8 +45,11 @@ def main(): event_id = event["id"] if event_id == "image": - arrow_image = event["value"][0] - encoding = arrow_image["encoding"].as_py() + storage = event["value"] + metadata = event["metadata"] + encoding = metadata["encoding"] + width = metadata["width"] + height = metadata["height"] if encoding == "bgr8": channels = 3 @@ -54,18 +57,11 @@ def main(): else: raise RuntimeError(f"Unsupported image encoding: {encoding}") - image = { - "width": np.uint32(arrow_image["width"].as_py()), - "height": np.uint32(arrow_image["height"].as_py()), - "encoding": encoding, - "channels": channels, - "data": arrow_image["data"].values.to_numpy().astype(storage_type), - } - - frame = image["data"].reshape( - (image["height"], image["width"], image["channels"]) + frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) ) - if encoding == "bgr8": frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) else: From dc0d2f25157f6a5d467530da99b545a1e205a025 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 23 Jul 2024 11:28:35 +0200 Subject: [PATCH 18/18] Refactor rerun example by using metadata from both image and bbox definition to make our input more generalistic. Rewrite README documentation to reflect metadata changes --- examples/rerun-viewer/dataflow.yml | 78 ++++---- examples/rerun-viewer/object_detection.py | 45 ----- examples/rerun-viewer/plot.py | 90 --------- examples/rerun-viewer/run.rs | 30 ++- examples/rerun-viewer/webcam.py | 56 ------ node-hub/dora-rerun/README.md | 12 +- node-hub/dora-rerun/src/main.rs | 172 ++++++++++++------ node-hub/opencv-plot/README.md | 47 +++-- node-hub/opencv-plot/opencv_plot/main.py | 53 ++++-- node-hub/opencv-video-capture/README.md | 40 ++-- .../opencv_video_capture/main.py | 15 +- node-hub/ultralytics-yolo/README.md | 67 ++++--- .../ultralytics-yolo/ultralytics_yolo/main.py | 24 ++- 13 files changed, 343 insertions(+), 386 deletions(-) delete mode 100755 examples/rerun-viewer/object_detection.py delete mode 100755 examples/rerun-viewer/plot.py delete mode 100755 examples/rerun-viewer/webcam.py diff --git a/examples/rerun-viewer/dataflow.yml b/examples/rerun-viewer/dataflow.yml index 5e179f02..1a484807 100644 --- a/examples/rerun-viewer/dataflow.yml +++ b/examples/rerun-viewer/dataflow.yml @@ -1,48 +1,38 @@ nodes: - - id: webcam - custom: - source: ./webcam.py - inputs: - tick: - source: dora/timer/millis/10 - queue_size: 1000 - outputs: - - image - - text - envs: - IMAGE_WIDTH: 960 - IMAGE_HEIGHT: 540 + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: dora/timer/millis/20 + outputs: + - image + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + ENCODING: rgb8 - - - id: object_detection - custom: - source: ./object_detection.py - inputs: - image: webcam/image - outputs: - - bbox - envs: - IMAGE_WIDTH: 960 - IMAGE_HEIGHT: 540 + - id: object-detection + build: pip install -e ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: + source: camera/image + queue_size: 1 + outputs: + - bbox + env: + MODEL: yolov8n.pt + FORMAT: xywh - id: rerun - custom: - source: dora-rerun - inputs: - image: webcam/image - text: webcam/text - boxes2d: object_detection/bbox - envs: - IMAGE_WIDTH: 960 - IMAGE_HEIGHT: 540 - IMAGE_DEPTH: 3 - - - id: matplotlib - custom: - source: ./plot.py - inputs: - image: webcam/image - bbox: object_detection/bbox - envs: - IMAGE_WIDTH: 960 - IMAGE_HEIGHT: 540 \ No newline at end of file + build: cargo build -p dora-rerun --release + path: dora-rerun + inputs: + image: + source: camera/image + queue_size: 1 + boxes2d: object-detection/bbox + env: + RERUN_FLUSH_TICK_SECS: "0.001" + RERUN_MEMORY_LIMIT: 25% diff --git a/examples/rerun-viewer/object_detection.py b/examples/rerun-viewer/object_detection.py deleted file mode 100755 index 2c606be9..00000000 --- a/examples/rerun-viewer/object_detection.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import cv2 -import numpy as np -from ultralytics import YOLO - -from dora import Node -import pyarrow as pa - -model = YOLO("yolov8n.pt") - -node = Node() - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", 960)) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", 540)) - -for event in node: - event_type = event["type"] - if event_type == "INPUT": - event_id = event["id"] - if event_id == "image": - print("[object detection] received image input") - image = event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - - frame = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) - results = model(frame) # includes NMS - # Process results - boxes = np.array(results[0].boxes.xywh.cpu()) - conf = np.array(results[0].boxes.conf.cpu()) - label = np.array(results[0].boxes.cls.cpu()) - # concatenate them together - arrays = np.concatenate((boxes, conf[:, None], label[:, None]), axis=1) - - node.send_output("bbox", pa.array(arrays.ravel()), event["metadata"]) - else: - print("[object detection] ignoring unexpected input:", event_id) - elif event_type == "STOP": - print("[object detection] received stop") - elif event_type == "ERROR": - print("[object detection] error: ", event["error"]) - else: - print("[object detection] received unexpected event:", event_type) diff --git a/examples/rerun-viewer/plot.py b/examples/rerun-viewer/plot.py deleted file mode 100755 index d6ec8389..00000000 --- a/examples/rerun-viewer/plot.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -from dora import Node -from dora import DoraStatus - -import cv2 -import numpy as np - -CI = os.environ.get("CI") - -font = cv2.FONT_HERSHEY_SIMPLEX - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", 960)) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", 540)) - - -class Plotter: - """ - Plot image and bounding box - """ - - def __init__(self): - self.image = [] - self.bboxs = [] - - def on_input( - self, - dora_input, - ) -> DoraStatus: - """ - Put image and bounding box on cv2 window. - - Args: - dora_input["id"] (str): Id of the dora_input declared in the yaml configuration - dora_input["value"] (arrow array): message of the dora_input - """ - if dora_input["id"] == "image": - image = ( - dora_input["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - ) - - image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - self.image = image - - elif dora_input["id"] == "bbox" and len(self.image) != 0: - bboxs = dora_input["value"].to_numpy() - self.bboxs = np.reshape(bboxs, (-1, 6)) - for bbox in self.bboxs: - [ - x, - y, - w, - h, - confidence, - label, - ] = bbox - cv2.rectangle( - self.image, - (int(x - w / 2), int(y - h / 2)), - (int(x + w / 2), int(y + h / 2)), - (0, 255, 0), - 2, - ) - - if CI != "true": - cv2.imshow("frame", self.image) - if cv2.waitKey(1) & 0xFF == ord("q"): - return DoraStatus.STOP - - return DoraStatus.CONTINUE - - -plotter = Plotter() -node = Node() - -for event in node: - event_type = event["type"] - if event_type == "INPUT": - status = plotter.on_input(event) - if status == DoraStatus.CONTINUE: - pass - elif status == DoraStatus.STOP: - print("plotter returned stop status") - break - elif event_type == "STOP": - print("received stop") - else: - print("received unexpected event:", event_type) diff --git a/examples/rerun-viewer/run.rs b/examples/rerun-viewer/run.rs index a14b553f..b575234d 100644 --- a/examples/rerun-viewer/run.rs +++ b/examples/rerun-viewer/run.rs @@ -1,5 +1,4 @@ use dora_core::{get_pip_path, get_python_path, run}; -use dora_download::download_file; use dora_tracing::set_up_tracing; use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; @@ -51,20 +50,13 @@ async fn main() -> eyre::Result<()> { ); } - run( - get_python_path().context("Could not get pip binary")?, - &["-m", "pip", "install", "--upgrade", "pip"], - None, - ) - .await - .context("failed to install pip")?; run( get_pip_path().context("Could not get pip binary")?, - &["install", "-r", "requirements.txt"], - None, + &["install", "maturin"], + Some(venv), ) .await - .context("pip install failed")?; + .context("pip install maturin failed")?; run( "maturin", @@ -73,12 +65,6 @@ async fn main() -> eyre::Result<()> { ) .await .context("maturin develop failed")?; - download_file( - "https://github.com/ultralytics/assets/releases/download/v0.0.0/yolov8n.pt", - Path::new("yolov8n.pt"), - ) - .await - .context("Could not download weights.")?; let dataflow = Path::new("dataflow.yml"); run_dataflow(dataflow).await?; @@ -88,6 +74,16 @@ async fn main() -> eyre::Result<()> { async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); + + // First build the dataflow (install requirements) + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--").arg("build").arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); diff --git a/examples/rerun-viewer/webcam.py b/examples/rerun-viewer/webcam.py deleted file mode 100755 index 33a7950d..00000000 --- a/examples/rerun-viewer/webcam.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import time -import numpy as np -import cv2 - -from dora import Node -import pyarrow as pa - -node = Node() - -IMAGE_INDEX = int(os.getenv("IMAGE_INDEX", 0)) -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", 960)) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", 540)) -video_capture = cv2.VideoCapture(IMAGE_INDEX) -video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, IMAGE_WIDTH) -video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, IMAGE_HEIGHT) -font = cv2.FONT_HERSHEY_SIMPLEX - -start = time.time() - -# Run for 20 seconds -while time.time() - start < 1000: - # Wait next dora_input - event = node.next() - if event is None: - break - - event_type = event["type"] - if event_type == "INPUT": - ret, frame = video_capture.read() - if not ret: - frame = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3), dtype=np.uint8) - cv2.putText( - frame, - "No Webcam was found at index %d" % (IMAGE_INDEX), - (int(30), int(30)), - font, - 0.75, - (255, 255, 255), - 2, - 1, - ) - if len(frame) != IMAGE_HEIGHT * IMAGE_WIDTH * 3: - print("frame size is not correct") - frame = cv2.resize(frame, (IMAGE_WIDTH, IMAGE_HEIGHT)) - - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - node.send_output( - "image", - pa.array(frame.ravel()), - event["metadata"], - ) - node.send_output("text", pa.array([f"send image at: {time.time()}"])) diff --git a/node-hub/dora-rerun/README.md b/node-hub/dora-rerun/README.md index 75973851..83c736e7 100644 --- a/node-hub/dora-rerun/README.md +++ b/node-hub/dora-rerun/README.md @@ -25,15 +25,15 @@ cargo install --git https://github.com/dora-rs/dora dora-rerun text: webcam/text boxes2d: object_detection/bbox envs: - IMAGE_WIDTH: 960 - IMAGE_HEIGHT: 540 - IMAGE_DEPTH: 3 RERUN_MEMORY_LIMIT: 25% ``` +## Input definition + +- image: UInt8Array + metadata { "width": int, "height": int, "encoding": str } +- boxes2D: StructArray + metadata { "format": str } +- text: StringArray + ## Configurations -- IMAGE_WIDTH: Image width in pixels -- IMAGE_HEIGHT: Image height in heights -- IMAGE_DEPTH: Image depth - RERUN_MEMORY_LIMIT: Rerun memory limit diff --git a/node-hub/dora-rerun/src/main.rs b/node-hub/dora-rerun/src/main.rs index 3bf8c231..d092e17e 100644 --- a/node-hub/dora-rerun/src/main.rs +++ b/node-hub/dora-rerun/src/main.rs @@ -3,12 +3,15 @@ use std::env::VarError; use dora_node_api::{ - arrow::array::{Float32Array, StringArray, UInt8Array}, - DoraNode, Event, + arrow::{ + array::{AsArray, StringArray, StructArray, UInt8Array}, + datatypes::Float32Type, + }, + DoraNode, Event, Parameter, }; -use eyre::{eyre, Context, Result}; +use eyre::{eyre, Context, ContextCompat, Result}; use rerun::{ - external::re_types::ArrowBuffer, SpawnOptions, TensorBuffer, TensorData, TensorDimension, + external::re_types::ArrowBuffer, SpawnOptions, TensorBuffer, TensorData, TensorDimension, Text, }; fn main() -> Result<()> { @@ -39,60 +42,65 @@ fn main() -> Result<()> { .context("Could not spawn rerun visualization")?; while let Some(event) = events.recv() { - if let Event::Input { - id, - data, - metadata: _, - } = event - { + if let Event::Input { id, data, metadata } = event { if id.as_str().contains("image") { + let height = + if let Some(Parameter::Integer(height)) = metadata.parameters.get("height") { + height + } else { + &480 + }; + let width = + if let Some(Parameter::Integer(width)) = metadata.parameters.get("width") { + width + } else { + &640 + }; + let encoding = if let Some(Parameter::String(encoding)) = + metadata.parameters.get("encoding") + { + encoding + } else { + "bgr8" + }; + let channels = if encoding == "bgr8" { 3 } else { 3 }; + let shape = vec![ TensorDimension { name: Some("height".into()), - size: std::env::var(format!("{}_HEIGHT", id.as_str().to_uppercase())) - .context(format!( - "Could not read {}_HEIGHT env variable for parsing the image", - id.as_str().to_uppercase() - ))? - .parse() - .context(format!( - "Could not parse env {}_HEIGHT", - id.as_str().to_uppercase() - ))?, + size: *height as u64, }, TensorDimension { name: Some("width".into()), - size: std::env::var(format!("{}_WIDTH", id.as_str().to_uppercase())) - .context(format!( - "Could not read {}_WIDTH env variable for parsing the image", - id.as_str().to_uppercase() - ))? - .parse() - .context(format!( - "Could not parse env {}_WIDTH", - id.as_str().to_uppercase() - ))?, + size: *width as u64, }, TensorDimension { name: Some("depth".into()), - size: std::env::var(format!("{}_DEPTH", id.as_str().to_uppercase())) - .context(format!( - "Could not read {}_DEPTH env variable for parsing the image", - id.as_str().to_uppercase() - ))? - .parse() - .context(format!( - "Could not parse env {}_DEPTH", - id.as_str().to_uppercase() - ))?, + size: channels as u64, }, ]; - let buffer: UInt8Array = data.to_data().into(); - let buffer: &[u8] = buffer.values(); - let buffer = TensorBuffer::U8(ArrowBuffer::from(buffer)); - let tensordata = TensorData::new(shape.clone(), buffer); - let image = rerun::Image::new(tensordata); + let image = if encoding == "bgr8" { + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: &[u8] = buffer.values(); + + // Transpose values from BGR to RGB + let buffer: Vec = + buffer.chunks(3).flat_map(|x| [x[2], x[1], x[0]]).collect(); + let buffer = TensorBuffer::U8(ArrowBuffer::from(buffer)); + let tensordata = TensorData::new(shape.clone(), buffer); + + rerun::Image::new(tensordata) + } else if encoding == "rgb8" { + let buffer: &UInt8Array = data.as_any().downcast_ref().unwrap(); + let buffer: &[u8] = buffer.values(); + let buffer = TensorBuffer::U8(ArrowBuffer::from(buffer)); + let tensordata = TensorData::new(shape.clone(), buffer); + + rerun::Image::new(tensordata) + } else { + unimplemented!("We haven't worked on additional encodings.") + }; rec.log(id.as_str(), &image) .context("could not log image")?; @@ -107,21 +115,73 @@ fn main() -> Result<()> { } })?; } else if id.as_str().contains("boxes2d") { - let buffer: Float32Array = data.to_data().into(); - let buffer: &[f32] = buffer.values(); + let bbox_struct: StructArray = data.to_data().into(); + let format = + if let Some(Parameter::String(format)) = metadata.parameters.get("format") { + format + } else { + "xyxy" + }; + + // Cast Bbox + let bbox_buffer = bbox_struct + .column_by_name("bbox") + .context("Did not find labels field within bbox struct")?; + let bbox = bbox_buffer + .as_list_opt::() + .context("Could not deserialize bbox as list")? + .values(); + let bbox = bbox + .as_primitive_opt::() + .context("Could not get bbox value as list")? + .values(); + + // Cast Labels + let labels_buffer = bbox_struct + .column_by_name("labels") + .context("Did not find labels field within bbox struct")?; + let labels = labels_buffer + .as_list_opt::() + .context("Could not deserialize labels as list")? + .values(); + let labels = labels + .as_string_opt::() + .context("Could not deserialize labels as string")?; + let labels: Vec = labels.iter().map(|x| Text::from(x.unwrap())).collect(); + + // Cast confidence + let conf_buffer = bbox_struct + .column_by_name("conf") + .context("Did not find conf field within bbox struct")?; + let conf = conf_buffer + .as_list_opt::() + .context("Could not deserialize conf as list")? + .values(); + let _conf = conf + .as_primitive_opt::() + .context("Could not deserialize conf as string")?; + let mut centers = vec![]; let mut sizes = vec![]; - let mut classes = vec![]; - buffer.chunks(6).for_each(|block| { - if let [x, y, w, h, _conf, cls] = block { - centers.push((*x, *y)); - sizes.push((*w, *h)); - classes.push(*cls as u16); - } - }); + + if format == "xywh" { + bbox.chunks(4).for_each(|block| { + if let [x, y, w, h] = block { + centers.push((*x, *y)); + sizes.push((*w, *h)); + } + }); + } else if format == "xyxy" { + bbox.chunks(4).for_each(|block| { + if let [min_x, min_y, max_x, max_y] = block { + centers.push(((max_x + min_x) / 2., (max_y + min_y) / 2.)); + sizes.push(((max_x - min_x), (max_y - min_y))); + } + }); + } rec.log( id.as_str(), - &rerun::Boxes2D::from_centers_and_sizes(centers, sizes).with_class_ids(classes), + &rerun::Boxes2D::from_centers_and_sizes(centers, sizes).with_labels(labels), ) .wrap_err("Could not log Boxes2D")?; } diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md index ee77c86c..0eeb4fa3 100644 --- a/node-hub/opencv-plot/README.md +++ b/node-hub/opencv-plot/README.md @@ -23,21 +23,36 @@ This node is used to plot a text and a list of bbox on a base image (ideal for o - `image`: Arrow array containing the base image ```python -image: { - "width": np.uint32, - "height": np.uint32, - "encoding": bytes, - "data": np.array # flattened image data +## Image data +image_data: UInt8Array # Example: pa.array(img.ravel()) +metadata = { + "width": 640, + "height": 480, + "encoding": str, # bgr8, rgb8 } -encoded_image = pa.array([image]) +## Example +node.send_output( + image_data, {"width": 640, "height": 480, "encoding": "bgr8"} + ) -decoded_image = { - "width": np.uint32(encoded_image[0]["width"]), - "height": np.uint32(encoded_image[0]["height"]), - "encoding": encoded_image[0]["encoding"].as_py(), - "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) -} +## Decoding +storage = event["value"] + +metadata = event["metadata"] +encoding = metadata["encoding"] +width = metadata["width"] +height = metadata["height"] + +if encoding == "bgr8": + channels = 3 + storage_type = np.uint8 + +frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) +) ``` - `bbox`: an arrow array containing the bounding boxes, confidence scores, and class names of the detected objects @@ -47,15 +62,15 @@ decoded_image = { bbox: { "bbox": np.array, # flattened array of bounding boxes "conf": np.array, # flat array of confidence scores - "names": np.array, # flat array of class names + "labels": np.array, # flat array of class names } -encoded_bbox = pa.array([bbox]) +encoded_bbox = pa.array([bbox], {"format": "xyxy"}) decoded_bbox = { - "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), + "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 4), "conf": encoded_bbox[0]["conf"].values.to_numpy(), - "names": encoded_bbox[0]["names"].values.to_numpy(zero_copy_only=False), + "labels": encoded_bbox[0]["labels"].values.to_numpy(zero_copy_only=False), } ``` diff --git a/node-hub/opencv-plot/opencv_plot/main.py b/node-hub/opencv-plot/opencv_plot/main.py index 7d8af16e..4dd7adca 100644 --- a/node-hub/opencv-plot/opencv_plot/main.py +++ b/node-hub/opencv-plot/opencv_plot/main.py @@ -16,7 +16,7 @@ class Plot: bboxes: dict = { "bbox": np.array([]), "conf": np.array([]), - "names": np.array([]), + "labels": np.array([]), } text: str = "" @@ -26,7 +26,7 @@ class Plot: def plot_frame(plot): - for bbox in zip(plot.bboxes["bbox"], plot.bboxes["conf"], plot.bboxes["names"]): + for bbox in zip(plot.bboxes["bbox"], plot.bboxes["conf"], plot.bboxes["labels"]): [ [min_x, min_y, max_x, max_y], confidence, @@ -139,26 +139,57 @@ def main(): if encoding == "bgr8": channels = 3 storage_type = np.uint8 + plot.frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) + .copy() # Copy So that we can add annotation on the image + ) + elif encoding == "rgb8": + channels = 3 + storage_type = np.uint8 + frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) + ) + + plot.frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) else: raise RuntimeError(f"Unsupported image encoding: {encoding}") - plot.frame = ( - storage.to_numpy() - .astype(storage_type) - .reshape((height, width, channels)) - .copy() # Copy So that we can add annotation on the image - ) - plot_frame(plot) if not RUNNER_CI: if cv2.waitKey(1) & 0xFF == ord("q"): break elif event_id == "bbox": arrow_bbox = event["value"][0] + bbox_format = event["metadata"]["format"] + + if bbox_format == "xyxy": + bbox = arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4) + elif bbox_format == "xywh": + original_bbox = arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4) + bbox = np.array( + [ + ( + x - w / 2, + y - h / 2, + x + w / 2, + y + h / 2, + ) + for [x, y, w, h] in original_bbox + ] + ) + else: + raise RuntimeError(f"Unsupported bbox format: {bbox_format}") + plot.bboxes = { - "bbox": arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4), + "bbox": bbox, "conf": arrow_bbox["conf"].values.to_numpy(), - "names": arrow_bbox["names"].values.to_numpy(zero_copy_only=False), + "labels": arrow_bbox["labels"].values.to_numpy( + zero_copy_only=False + ), } elif event_id == "text": plot.text = event["value"][0].as_py() diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md index 923b9400..f7a6c230 100644 --- a/node-hub/opencv-video-capture/README.md +++ b/node-hub/opencv-video-capture/README.md @@ -29,22 +29,36 @@ This node is used to capture video from a camera using OpenCV. - `image`: an arrow array containing the captured image ```Python - -image: { - "width": np.uint32, - "height": np.uint32, - "encoding": str, - "data": np.array # flattened image data +## Image data +image_data: UInt8Array # Example: pa.array(img.ravel()) +metadata = { + "width": 640, + "height": 480, + "encoding": str, # bgr8, rgb8 } -encoded_image = pa.array([image]) +## Example +node.send_output( + image_data, {"width": 640, "height": 480, "encoding": "bgr8"} + ) -decoded_image = { - "width": np.uint32(encoded_image[0]["width"]), - "height": np.uint32(encoded_image[0]["height"]), - "encoding": encoded_image[0]["encoding"].as_py(), - "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) -} +## Decoding +storage = event["value"] + +metadata = event["metadata"] +encoding = metadata["encoding"] +width = metadata["width"] +height = metadata["height"] + +if encoding == "bgr8": + channels = 3 + storage_type = np.uint8 + +frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) +) ``` ## License diff --git a/node-hub/opencv-video-capture/opencv_video_capture/main.py b/node-hub/opencv-video-capture/opencv_video_capture/main.py index d9dee5e1..0f4e29ab 100644 --- a/node-hub/opencv-video-capture/opencv_video_capture/main.py +++ b/node-hub/opencv-video-capture/opencv_video_capture/main.py @@ -50,6 +50,7 @@ def main(): args = parser.parse_args() video_capture_path = os.getenv("CAPTURE_PATH", args.path) + encoding = os.getenv("ENCODING", "bgr8") if isinstance(video_capture_path, str) and video_capture_path.isnumeric(): video_capture_path = int(video_capture_path) @@ -102,15 +103,25 @@ def main(): ) # resize the frame - if image_width is not None and image_height is not None: + if ( + image_width is not None + and image_height is not None + and ( + frame.shape[1] != image_width or frame.shape[0] != image_height + ) + ): frame = cv2.resize(frame, (image_width, image_height)) + # Get the right encoding + if encoding == "rgb8": + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + storage = pa.array(frame.ravel()) metadata = event["metadata"] metadata["width"] = int(frame.shape[1]) metadata["height"] = int(frame.shape[0]) - metadata["encoding"] = "bgr8" + metadata["encoding"] = encoding node.send_output("image", storage, metadata) diff --git a/node-hub/ultralytics-yolo/README.md b/node-hub/ultralytics-yolo/README.md index fae47540..56703531 100644 --- a/node-hub/ultralytics-yolo/README.md +++ b/node-hub/ultralytics-yolo/README.md @@ -5,16 +5,16 @@ This node is used to detect objects in images using YOLOv8. # YAML ```yaml - - id: object_detection - build: pip install ../../node-hub/ultralytics-yolo - path: ultralytics-yolo - inputs: - image: webcam/image - - outputs: - - bbox - env: - MODEL: yolov5n.pt +- id: object_detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: webcam/image + + outputs: + - bbox + env: + MODEL: yolov5n.pt ``` # Inputs @@ -22,21 +22,36 @@ This node is used to detect objects in images using YOLOv8. - `image`: Arrow array containing the base image ```python -image: { - "width": np.uint32, - "height": np.uint32, - "encoding": str, - "data": np.array # flattened image data +## Image data +image_data: UInt8Array # Example: pa.array(img.ravel()) +metadata = { + "width": 640, + "height": 480, + "encoding": str, # bgr8, rgb8 } -encoded_image = pa.array([image]) +## Example +node.send_output( + image_data, {"width": 640, "height": 480, "encoding": "bgr8"} + ) -decoded_image = { - "width": np.uint32(encoded_image[0]["width"]), - "height": np.uint32(encoded_image[0]["height"]), - "encoding": encoded_image[0]["encoding"].as_py(), - "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) -} +## Decoding +storage = event["value"] + +metadata = event["metadata"] +encoding = metadata["encoding"] +width = metadata["width"] +height = metadata["height"] + +if encoding == "bgr8": + channels = 3 + storage_type = np.uint8 + +frame = ( + storage.to_numpy() + .astype(storage_type) + .reshape((height, width, channels)) +) ``` @@ -49,15 +64,15 @@ decoded_image = { bbox: { "bbox": np.array, # flattened array of bounding boxes "conf": np.array, # flat array of confidence scores - "names": np.array, # flat array of class names + "labels": np.array, # flat array of class names } -encoded_bbox = pa.array([bbox]) +encoded_bbox = pa.array([bbox], {"format": "xyxy"}) decoded_bbox = { - "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), + "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 4), "conf": encoded_bbox[0]["conf"].values.to_numpy(), - "names": encoded_bbox[0]["names"].values.to_numpy(zero_copy_only=False), + "labels": encoded_bbox[0]["labels"].values.to_numpy(zero_copy_only=False), } ``` diff --git a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py index 23ca85b0..42ef980f 100644 --- a/node-hub/ultralytics-yolo/ultralytics_yolo/main.py +++ b/node-hub/ultralytics-yolo/ultralytics_yolo/main.py @@ -32,6 +32,7 @@ def main(): args = parser.parse_args() model_path = os.getenv("MODEL", args.model) + bbox_format = os.getenv("FORMAT", "xyxy") model = YOLO(model_path) node = Node(args.name) @@ -54,6 +55,9 @@ def main(): if encoding == "bgr8": channels = 3 storage_type = np.uint8 + elif encoding == "rgb8": + channels = 3 + storage_type = np.uint8 else: raise RuntimeError(f"Unsupported image encoding: {encoding}") @@ -64,12 +68,20 @@ def main(): ) if encoding == "bgr8": frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + elif encoding == "rgb8": + pass else: raise RuntimeError(f"Unsupported image encoding: {encoding}") results = model(frame, verbose=False) # includes NMS - bboxes = np.array(results[0].boxes.xyxy.cpu()) + if bbox_format == "xyxy": + bboxes = np.array(results[0].boxes.xyxy.cpu()) + elif bbox_format == "xywh": + bboxes = np.array(results[0].boxes.xywh.cpu()) + else: + raise RuntimeError(f"Unsupported bbox format: {bbox_format}") + conf = np.array(results[0].boxes.conf.cpu()) labels = np.array(results[0].boxes.cls.cpu()) @@ -78,13 +90,17 @@ def main(): bbox = { "bbox": bboxes.ravel(), "conf": conf, - "names": names, + "labels": names, } + bbox = pa.array([bbox]) + + metadata = event["metadata"] + metadata["format"] = bbox_format node.send_output( "bbox", - pa.array([bbox]), - event["metadata"], + bbox, + metadata, ) elif event_type == "ERROR":