From 05f2d3fc9f14db529863246ccd18268ffc2231c1 Mon Sep 17 00:00:00 2001 From: Hennzau Date: Tue, 9 Jul 2024 19:23:55 +0200 Subject: [PATCH] nodes_hub -> node-hub + opencv-plot and opencv-video-capture .idea .idea README.md and better outputs/inputs README.md and ultralytics-yolo fix queue_size and dynamic README.md Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI Fix CI --- .github/workflows/ci.yml | 7 +- Cargo.toml | 4 +- examples/python-dataflow/README.md | 33 +++- examples/python-dataflow/dataflow.yml | 45 +++-- examples/python-dataflow/dataflow_dynamic.yml | 45 ++++- examples/python-dataflow/plot_dynamic.py | 71 ------- examples/python-dataflow/run.rs | 20 +- node-hub/README.md | 88 +++++++++ .../dora-record/Cargo.toml | 0 {nodes_hub => node-hub}/dora-record/README.md | 0 .../dora-record/src/main.rs | 0 {nodes_hub => node-hub}/dora-rerun/Cargo.toml | 0 {nodes_hub => node-hub}/dora-rerun/README.md | 0 .../dora-rerun/src/main.rs | 0 node-hub/opencv-plot/README.md | 80 ++++++++ node-hub/opencv-plot/main.py | 179 ++++++++++++++++++ node-hub/opencv-plot/pyproject.toml | 25 +++ node-hub/opencv-video-capture/README.md | 53 ++++++ node-hub/opencv-video-capture/main.py | 96 ++++++++++ node-hub/opencv-video-capture/pyproject.toml | 25 +++ node-hub/ultralytics-yolo/README.md | 66 +++++++ node-hub/ultralytics-yolo/main.py | 158 ++++++++++++++++ node-hub/ultralytics-yolo/pyproject.toml | 25 +++ nodes_hub/opencv-plot/plot.py | 73 ------- nodes_hub/opencv-plot/requirements.txt | 2 - .../opencv-video-capture/requirements.txt | 3 - .../opencv-video-capture/video_capture.py | 52 ----- nodes_hub/ultralytics-yolo/requirements.txt | 3 - nodes_hub/ultralytics-yolo/yolo.py | 50 ----- 29 files changed, 915 insertions(+), 288 deletions(-) delete mode 100644 examples/python-dataflow/plot_dynamic.py create mode 100644 node-hub/README.md rename {nodes_hub => node-hub}/dora-record/Cargo.toml (100%) rename {nodes_hub => node-hub}/dora-record/README.md (100%) rename {nodes_hub => node-hub}/dora-record/src/main.rs (100%) rename {nodes_hub => node-hub}/dora-rerun/Cargo.toml (100%) rename {nodes_hub => node-hub}/dora-rerun/README.md (100%) rename {nodes_hub => node-hub}/dora-rerun/src/main.rs (100%) create mode 100644 node-hub/opencv-plot/README.md create mode 100644 node-hub/opencv-plot/main.py create mode 100644 node-hub/opencv-plot/pyproject.toml create mode 100644 node-hub/opencv-video-capture/README.md create mode 100644 node-hub/opencv-video-capture/main.py create mode 100644 node-hub/opencv-video-capture/pyproject.toml create mode 100644 node-hub/ultralytics-yolo/README.md create mode 100644 node-hub/ultralytics-yolo/main.py create mode 100644 node-hub/ultralytics-yolo/pyproject.toml delete mode 100644 nodes_hub/opencv-plot/plot.py delete mode 100644 nodes_hub/opencv-plot/requirements.txt delete mode 100644 nodes_hub/opencv-video-capture/requirements.txt delete mode 100644 nodes_hub/opencv-video-capture/video_capture.py delete mode 100644 nodes_hub/ultralytics-yolo/requirements.txt delete mode 100644 nodes_hub/ultralytics-yolo/yolo.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8708ba96..71501d15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -315,14 +315,15 @@ jobs: cd test_python_project dora up dora list + dora build dataflow.yml dora start dataflow.yml --name ci-python-test --detach sleep 10 dora stop --name ci-python-test --grace-duration 5s - pip install "numpy<2.0.0" opencv-python + dora build ../examples/python-dataflow/dataflow_dynamic.yml dora start ../examples/python-dataflow/dataflow_dynamic.yml --name ci-python-dynamic --detach - python ../examples/python-dataflow/plot_dynamic.py + ultralytics-yolo --name object-detection sleep 5 - dora stop --name ci-python-test --grace-duration 5s + dora stop --name ci-python-dynamic --grace-duration 5s dora destroy - name: "Test CLI (C)" diff --git a/Cargo.toml b/Cargo.toml index fa52a3b6..41fc4c5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,8 @@ members = [ "libraries/shared-memory-server", "libraries/extensions/download", "libraries/extensions/telemetry/*", - "nodes_hub/dora-record", - "nodes_hub/dora-rerun", + "node-hub/dora-record", + "node-hub/dora-rerun", "libraries/extensions/ros2-bridge", "libraries/extensions/ros2-bridge/msg-gen", "libraries/extensions/ros2-bridge/python", diff --git a/examples/python-dataflow/README.md b/examples/python-dataflow/README.md index 2211a721..8e2c75ec 100644 --- a/examples/python-dataflow/README.md +++ b/examples/python-dataflow/README.md @@ -1,25 +1,44 @@ # Python Dataflow Example -This examples shows how to create and connect dora operators and custom nodes in Python. +This examples shows how to create and connect dora nodes in Python. ## Overview The [`dataflow.yml`](./dataflow.yml) defines a simple dataflow graph with the following three nodes: - a webcam node, that connects to your webcam and feed the dataflow with webcam frame as jpeg compressed bytearray. -- an object detection node, that apply Yolo v5 on the webcam image. The model is imported from Pytorch Hub. The output is the bounding box of each object detected, the confidence and the class. You can have more info here: https://pytorch.org/hub/ultralytics_yolov5/ +- an object detection node, that apply Yolo v5 on the webcam image. The model is imported from Pytorch Hub. The output + is the bounding box of each object detected, the confidence and the associated label. You can have more info + here: https://pytorch.org/hub/ultralytics_yolov5/ - a window plotting node, that will retrieve the webcam image and the Yolov5 bounding box and join the two together. +The same dataflow is implemented for a `dynamic-node` in [`dataflow_dynamic.yml`](./dataflow_dynamic.yml). It contains +the same nodes as the previous dataflow, but the object detection node is a dynamic node. See the next section for more +information on how to start such a dataflow. + ## Getting started +After installing Rust, `dora-cli` and `Python >3.11`, you will need to **activate** (or create and **activate**) a +virtual +environment. Then, you will need to install the dependencies: + ```bash -cargo run --example python-dataflow +cd examples/python-dataflow +dora build ./dataflow.yml (or dora build ./dataflow_dynamic.yml) ``` -## Run the dataflow as a standalone +It will install the required dependencies for the Python nodes. -- Start the `dora-daemon`: +Then you can run the dataflow: +```bash +dora up +dora start ./dataflow.yml (or dora start ./dataflow_dynamic.yml) ``` -../../target/release/dora-daemon --run-dataflow dataflow.yml -``` + +**Note**: if you're running the dynamic dataflow, you will need to start manually the ultralytics-yolo node: + +```bash +# activate your virtual environment in another terminal +python ultralytics-yolo --name object-detection --model yolov5n.pt +``` \ No newline at end of file diff --git a/examples/python-dataflow/dataflow.yml b/examples/python-dataflow/dataflow.yml index 2b608ce7..23562fed 100644 --- a/examples/python-dataflow/dataflow.yml +++ b/examples/python-dataflow/dataflow.yml @@ -1,25 +1,44 @@ nodes: - - id: webcam - build: pip install -r ../../nodes_hub/opencv-video-capture/requirements.txt - path: ../../nodes_hub/opencv-video-capture/video_capture.py + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture inputs: - tick: dora/timer/millis/50 + tick: plot/tick + outputs: - image + env: - DURATION: 100 + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 - - id: object_detection - build: pip install -r ../../nodes_hub/ultralytics-yolo/requirements.txt - path: ../../nodes_hub/ultralytics-yolo/yolo.py + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo inputs: - image: webcam/image + image: + source: camera/image + queue_size: 1 + outputs: - bbox + env: + MODEL: yolov5n.pt - id: plot - build: pip install -r ../../nodes_hub/opencv-plot/requirements.txt - path: ../../nodes_hub/opencv-plot/plot.py + build: pip install ../../node-hub/opencv-plot + path: opencv-plot inputs: - image: webcam/image - bbox: object_detection/bbox + image: + source: camera/image + queue_size: 1 + + bbox: object-detection/bbox + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick \ No newline at end of file diff --git a/examples/python-dataflow/dataflow_dynamic.yml b/examples/python-dataflow/dataflow_dynamic.yml index 3c9ae1cb..140cd6c8 100644 --- a/examples/python-dataflow/dataflow_dynamic.yml +++ b/examples/python-dataflow/dataflow_dynamic.yml @@ -1,15 +1,44 @@ nodes: - - id: webcam - build: pip install -r ../../nodes_hub/opencv-webcam/requirements.txt - path: ../../nodes_hub/opencv-webcam/webcam.py + - id: camera + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture inputs: - tick: - source: dora/timer/millis/50 - queue_size: 1000 + tick: plot/tick + outputs: - image - - id: plot + env: + CAPTURE_PATH: 0 + IMAGE_WIDTH: 640 + IMAGE_HEIGHT: 480 + + - id: object-detection + build: pip install ../../node-hub/ultralytics-yolo path: dynamic inputs: - image: webcam/image + image: + source: camera/image + queue_size: 1 + + outputs: + - bbox + env: + MODEL: yolov5n.pt + + - id: plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + image: + source: camera/image + queue_size: 1 + + bbox: object-detection/bbox + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick \ No newline at end of file diff --git a/examples/python-dataflow/plot_dynamic.py b/examples/python-dataflow/plot_dynamic.py deleted file mode 100644 index 31c76ee2..00000000 --- a/examples/python-dataflow/plot_dynamic.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -from dataclasses import dataclass - -import cv2 -import numpy as np - -from dora import Node - -CI = os.environ.get("CI") - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) - -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -@dataclass -class Plotter: - frame: np.array = np.array([]) - bboxes: np.array = np.array([]) - - -if __name__ == "__main__": - plotter = Plotter() - node = Node("plot") - - for event in node: - event_type = event["type"] - if event_type == "INPUT": - if event["id"] == "image": - frame = event["value"].to_numpy() - frame = ( - event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - ) - plotter.frame = frame - - elif event["id"] == "bbox" and len(plotter.frame) != 0: - bboxs = event["value"].to_numpy() - plotter.bboxes = np.reshape(bboxs, (-1, 6)) - for bbox in plotter.bboxs: - [ - min_x, - min_y, - max_x, - max_y, - confidence, - label, - ] = bbox - cv2.rectangle( - plotter.frame, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - plotter.frame, - LABELS[int(label)] + f", {confidence:0.2f}", - (int(max_x), int(max_y)), - FONT, - 0.75, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", plotter.frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - break diff --git a/examples/python-dataflow/run.rs b/examples/python-dataflow/run.rs index 20fdc399..25d0f870 100644 --- a/examples/python-dataflow/run.rs +++ b/examples/python-dataflow/run.rs @@ -1,4 +1,4 @@ -use dora_core::{get_pip_path, get_python_path, run}; +use dora_core::{get_python_path, run}; use dora_tracing::set_up_tracing; use eyre::{bail, ContextCompat, WrapErr}; use std::path::Path; @@ -50,6 +50,14 @@ async fn main() -> eyre::Result<()> { ); } + run( + "pip", + &["install", "maturin"], + Some (venv), + ) + .await + .context("pip install maturin failed")?; + run( "maturin", &["develop"], @@ -66,6 +74,16 @@ async fn main() -> eyre::Result<()> { async fn run_dataflow(dataflow: &Path) -> eyre::Result<()> { let cargo = std::env::var("CARGO").unwrap(); + + // First build the dataflow (install requirements) + let mut cmd = tokio::process::Command::new(&cargo); + cmd.arg("run"); + cmd.arg("--package").arg("dora-cli"); + cmd.arg("--").arg("build").arg(dataflow); + if !cmd.status().await?.success() { + bail!("failed to run dataflow"); + }; + let mut cmd = tokio::process::Command::new(&cargo); cmd.arg("run"); cmd.arg("--package").arg("dora-cli"); diff --git a/node-hub/README.md b/node-hub/README.md new file mode 100644 index 00000000..9e59bc16 --- /dev/null +++ b/node-hub/README.md @@ -0,0 +1,88 @@ +## Dora Node Hub + +This hub contains useful pre-built nodes for Dora. + +# Structure + +The structure of the node hub is as follows (please use the same structure if you need to add a new node): + +``` +node-hub/ +└── your-node/ + ├── main.py + ├── README.mdr + └── pyproject.toml +``` + +The idea is to make a `pyproject.toml` file that will install the required dependencies for the node **and** attach main +function of the node inside a callable script in your environment. + +To do so, you will need to add a `main` function inside the `main.py` file. + +```python +def main(): + pass +``` + +And then you will need to adapt the following `pyproject.toml` file: + +```toml +[tool.poetry] +name = "[name of the node e.g. video-encoder, with '-' to replace spaces]" +version = "0.1" +authors = ["[Pseudo/Name] <[email]>"] +description = "Dora Node for []" +readme = "README.md" + +packages = [ + { include = "main.py", to = "[name of the node with '_' to replace spaces]" } +] + +[tool.poetry.dependencies] +python = "^3.11" +dora-rs = "0.3.5" +... [add your dependencies here] ... + +[tool.poetry.scripts] +[name of the node with '-' to replace spaces] = "[name of the node with '_' to replace spaces].main:main" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +``` + +Finally, the README.md file should explicit all inputs/outputs of the node and how to configure it in the YAML file. + +# Example + +```toml +[tool.poetry] +name = "opencv-plot" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for plotting data with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_plot" } +] + +[tool.poetry.dependencies] +python = "^3.11" +dora-rs = "^0.3.5" +opencv-python = "^4.10.0.84" + +[tool.poetry.scripts] +opencv-plot = "opencv_plot.main:main" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../NOTICE.md) for more information. diff --git a/nodes_hub/dora-record/Cargo.toml b/node-hub/dora-record/Cargo.toml similarity index 100% rename from nodes_hub/dora-record/Cargo.toml rename to node-hub/dora-record/Cargo.toml diff --git a/nodes_hub/dora-record/README.md b/node-hub/dora-record/README.md similarity index 100% rename from nodes_hub/dora-record/README.md rename to node-hub/dora-record/README.md diff --git a/nodes_hub/dora-record/src/main.rs b/node-hub/dora-record/src/main.rs similarity index 100% rename from nodes_hub/dora-record/src/main.rs rename to node-hub/dora-record/src/main.rs diff --git a/nodes_hub/dora-rerun/Cargo.toml b/node-hub/dora-rerun/Cargo.toml similarity index 100% rename from nodes_hub/dora-rerun/Cargo.toml rename to node-hub/dora-rerun/Cargo.toml diff --git a/nodes_hub/dora-rerun/README.md b/node-hub/dora-rerun/README.md similarity index 100% rename from nodes_hub/dora-rerun/README.md rename to node-hub/dora-rerun/README.md diff --git a/nodes_hub/dora-rerun/src/main.rs b/node-hub/dora-rerun/src/main.rs similarity index 100% rename from nodes_hub/dora-rerun/src/main.rs rename to node-hub/dora-rerun/src/main.rs diff --git a/node-hub/opencv-plot/README.md b/node-hub/opencv-plot/README.md new file mode 100644 index 00000000..95017cbb --- /dev/null +++ b/node-hub/opencv-plot/README.md @@ -0,0 +1,80 @@ +# Dora Node for plotting data with OpenCV + +This node is used to plot a text and a list of bbox on a base image (ideal for object detection). + +# YAML + +```yaml + - id: opencv-plot + build: pip install ../../node-hub/opencv-plot + path: opencv-plot + inputs: + # image: Arrow array of size 1 containing the base image + # bbox: Arrow array of bbox + # text: Arrow array of size 1 containing the text to be plotted + + tick: + source: dora/timer/millis/16 # this node display a window, so it's better to deflect the timer, so when the window is closed, the ticks are not sent anymore in the graph + queue_size: 1 + + outputs: + - tick + + env: + PLOT_WIDTH: 640 # optional, default is image input width + PLOT_HEIGHT: 480 # optional, default is image input height +``` + +# Inputs +- +- `tick`: empty Arrow array to trigger the capture + +- `image`: Arrow array containing the base image + +```python +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} + +``` + +- `text`: Arrow array containing the text to be plotted + +```python +text = { + "text": str, + "font_scale": np.float32, + "color": (np.uint8, np.uint8, np.uint8), + "thickness": np.uint32, + "position": (np.uint32, np.uint32) +} + +encoded_text = pa.array([text]) + +decoded_text = { + "text": encoded_text[0]["text"].as_py(), + "font_scale": np.float32(encoded_text[0]["font_scale"].as_py()), + "color": (np.uint8(encoded_text[0]["color"].as_py()[0]), + np.uint8(encoded_text[0]["color"].as_py()[1]), + np.uint8(encoded_text[0]["color"].as_py()[2])), + "thickness": np.uint32(encoded_text[0]["thickness"].as_py()), + "position": (np.uint32(encoded_text[0]["position"].as_py()[0]), + np.uint32(encoded_text[0]["position"].as_py()[1])) +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/opencv-plot/main.py b/node-hub/opencv-plot/main.py new file mode 100644 index 00000000..ec004701 --- /dev/null +++ b/node-hub/opencv-plot/main.py @@ -0,0 +1,179 @@ +import os +import argparse +import cv2 + +import numpy as np +import pyarrow as pa + +from dora import Node + + +class Plot: + frame: np.array = np.array([]) + + bboxes: {} = { + "bbox": np.array([]), + "conf": np.array([]), + "names": np.array([]), + } + + text: {} = { + "text": "", + "font_scale": np.float32(0.0), + "color": (np.uint8(0), np.uint8(0), np.uint8(0)), + "thickness": np.uint32(0), + "position": (np.uint32(0), np.uint32(0)), + } + + width: np.uint32 = None + height: np.uint32 = None + + +def plot_frame(plot, ci_enabled): + for bbox in zip(plot.bboxes["bbox"], plot.bboxes["conf"], plot.bboxes["names"]): + [ + [min_x, min_y, max_x, max_y], + confidence, + label, + ] = bbox + cv2.rectangle( + plot.frame, + (int(min_x), int(min_y)), + (int(max_x), int(max_y)), + (0, 255, 0), + 2, + ) + + cv2.putText( + plot.frame, + f"{label}, {confidence:0.2f}", + (int(max_x) - 120, int(max_y) - 10), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (0, 255, 0), + 1, + 1, + ) + + cv2.putText( + plot.frame, + plot.text["text"], + (int(plot.text["position"][0]), int(plot.text["position"][1])), + cv2.FONT_HERSHEY_SIMPLEX, + float(plot.text["font_scale"]), + (int(plot.text["color"][0]), int(plot.text["color"][1]), int(plot.text["color"][2])), + int(plot.text["thickness"]), + 1, + ) + + if plot.width is not None and plot.height is not None: + plot.frame = cv2.resize(plot.frame, (plot.width, plot.height)) + + if not ci_enabled: + if len(plot.frame.shape) >= 3: + cv2.imshow("Dora Node: opencv-plot", plot.frame) + + if cv2.waitKey(1) & 0xFF == ord('q'): + return True + + return False + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="OpenCV Plotter: This node is used to plot text and bounding boxes on an image.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="opencv-plot") + parser.add_argument("--plot-width", type=int, required=False, help="The width of the plot.", default=None) + parser.add_argument("--plot-height", type=int, required=False, help="The height of the plot.", default=None) + + args = parser.parse_args() + + plot_width = os.getenv("PLOT_WIDTH", args.plot_width) + plot_height = os.getenv("PLOT_HEIGHT", args.plot_height) + + if plot_width is not None: + if isinstance(plot_width, str) and plot_width.isnumeric(): + plot_width = int(plot_width) + + if plot_height is not None: + if isinstance(plot_height, str) and plot_height.isnumeric(): + plot_height = int(plot_height) + + # check if the code is running in a CI environment (e.g. GitHub Actions) (parse to bool) + ci_enabled = os.getenv("CI", False) + if ci_enabled == "true": + ci_enabled = True + + node = Node(args.name) # provide the name to connect to the dataflow if dynamic node + plot = Plot() + + plot.width = plot_width + plot.height = plot_height + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "tick": + node.send_output( + "tick", + pa.array([]), + event["metadata"] + ) + + elif event_id == "image": + arrow_image = event["value"][0] + image = { + "width": np.uint32(arrow_image["width"].as_py()), + "height": np.uint32(arrow_image["height"].as_py()), + "channels": np.uint8(arrow_image["channels"].as_py()), + "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + } + + plot.frame = np.reshape(image["data"], (image["height"], image["width"], image["channels"])) + + if plot_frame(plot, ci_enabled): + break + + elif event_id == "bbox": + arrow_bbox = event["value"][0] + plot.bboxes = { + "bbox": arrow_bbox["bbox"].values.to_numpy().reshape(-1, 4), + "conf": arrow_bbox["conf"].values.to_numpy(), + "names": arrow_bbox["names"].values.to_pylist(), + } + + if plot_frame(plot, ci_enabled): + break + + elif event_id == "text": + arrow_text = event["value"][0] + plot.text = { + "text": arrow_text["text"].as_py(), + "font_scale": np.float32(arrow_text["font_scale"].as_py()), + "color": (np.uint8(arrow_text["color"].as_py()[0]), + np.uint8(arrow_text["color"].as_py()[1]), + np.uint8(arrow_text["color"].as_py()[2])), + "thickness": np.uint32(arrow_text["thickness"].as_py()), + "position": (np.uint32(arrow_text["position"].as_py()[0]), + np.uint32(arrow_text["position"].as_py()[1])) + } + + if plot_frame(plot, ci_enabled): + break + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/opencv-plot/pyproject.toml b/node-hub/opencv-plot/pyproject.toml new file mode 100644 index 00000000..a1f01c45 --- /dev/null +++ b/node-hub/opencv-plot/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "opencv-plot" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for plotting text and bbox on image with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_plot" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +opencv-python = ">= 4.1.1" + +[tool.poetry.scripts] +opencv-plot = "opencv_plot.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/node-hub/opencv-video-capture/README.md b/node-hub/opencv-video-capture/README.md new file mode 100644 index 00000000..f4f10cdd --- /dev/null +++ b/node-hub/opencv-video-capture/README.md @@ -0,0 +1,53 @@ +# Dora Node for capturing video with OpenCV + +This node is used to capture video from a camera using OpenCV. + +# YAML + +```yaml + - id: opencv-video-capture + build: pip install ../../node-hub/opencv-video-capture + path: opencv-video-capture + inputs: + tick: dora/timer/millis/16 # try to capture at 60fps + + outputs: + - image: # the captured image + + env: + PATH: 0 # optional, default is 0 + + IMAGE_WIDTH: 640 # optional, default is video capture width + IMAGE_HEIGHT: 480 # optional, default is video capture height +``` + +# Inputs + +- `tick`: empty Arrow array to trigger the capture + +# Outputs + +- `image`: an arrow array containing the captured image + +```Python + +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/opencv-video-capture/main.py b/node-hub/opencv-video-capture/main.py new file mode 100644 index 00000000..5160e0fb --- /dev/null +++ b/node-hub/opencv-video-capture/main.py @@ -0,0 +1,96 @@ +import os +import argparse +import cv2 + +import numpy as np +import pyarrow as pa + +from dora import Node + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="OpenCV Video Capture: This node is used to capture video from a camera.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="opencv-video-capture") + parser.add_argument("--path", type=int, required=False, + help="The path of the device to capture (e.g. /dev/video1, or an index like 0, 1...", default=0) + parser.add_argument("--image-width", type=int, required=False, + help="The width of the image output. Default is the camera width.", default=None) + parser.add_argument("--image-height", type=int, required=False, + help="The height of the camera. Default is the camera height.", default=None) + + args = parser.parse_args() + + video_capture_path = os.getenv("CAPTURE_PATH", args.path) + + if isinstance(video_capture_path, str) and video_capture_path.isnumeric(): + video_capture_path = int(video_capture_path) + + print(type(video_capture_path)) + + image_width = os.getenv("IMAGE_WIDTH", args.image_width) + image_height = os.getenv("IMAGE_HEIGHT", args.image_height) + + if image_width is not None: + if isinstance(image_width, str) and image_width.isnumeric(): + image_width = int(image_width) + + if image_height is not None: + if isinstance(image_height, str) and image_height.isnumeric(): + image_height = int(image_height) + + video_capture = cv2.VideoCapture(video_capture_path) + node = Node(args.name) + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "tick": + ret, frame = video_capture.read() + + if not ret: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + cv2.putText( + frame, + f'Error: Could not read frame from camera at path {video_capture_path}.', + (int(30), int(30)), + cv2.FONT_HERSHEY_SIMPLEX, + 0.50, + (255, 255, 255), + 1, + 1, + ) + + # resize the frame + if image_width is not None and image_height is not None: + frame = cv2.resize(frame, (image_width, image_height)) + + image = { + "width": np.uint32(frame.shape[1]), + "height": np.uint32(frame.shape[0]), + "channels": np.uint8(frame.shape[2]), + "data": frame.ravel() + } + + node.send_output( + "image", + pa.array([image]), + event["metadata"] + ) + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/opencv-video-capture/pyproject.toml b/node-hub/opencv-video-capture/pyproject.toml new file mode 100644 index 00000000..00100acf --- /dev/null +++ b/node-hub/opencv-video-capture/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "opencv-video-capture" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for capturing video with OpenCV" +readme = "README.md" + +packages = [ + { include = "main.py", to = "opencv_video_capture" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +opencv-python = ">= 4.1.1" + +[tool.poetry.scripts] +opencv-video-capture = "opencv_video_capture.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/node-hub/ultralytics-yolo/README.md b/node-hub/ultralytics-yolo/README.md new file mode 100644 index 00000000..5a66c832 --- /dev/null +++ b/node-hub/ultralytics-yolo/README.md @@ -0,0 +1,66 @@ +# Dora Node for detecting objects in images using YOLOv8 + +This node is used to detect objects in images using YOLOv8. + +# YAML + +```yaml + - id: object_detection + build: pip install ../../node-hub/ultralytics-yolo + path: ultralytics-yolo + inputs: + image: webcam/image + + outputs: + - bbox + env: + MODEL: yolov5n.pt +``` + +# Inputs + +- `image`: Arrow array containing the base image + +```python +image = { + "width": np.uint32, + "height": np.uint32, + "channels": np.uint8, + "data": np.array # flattened image data +} + +encoded_image = pa.array([image]) + +decoded_image = { + "width": np.uint32(encoded_image[0]["width"].as_py()), + "height": np.uint32(encoded_image[0]["height"].as_py()), + "channels": np.uint8(encoded_image[0]["channels"].as_py()), + "data": encoded_image[0]["data"].values.to_numpy().astype(np.uint8) +} + +``` + +# Outputs + +- `bbox`: an arrow array containing the bounding boxes, confidence scores, and class names of the detected objects + +```Python + +bbox = { + "bbox": np.array, # flattened array of bounding boxes + "conf": np.array, # flat array of confidence scores + "names": np.array, # flat array of class names +} + +encoded_bbox = pa.array([bbox]) + +decoded_bbox = { + "bbox": encoded_bbox[0]["bbox"].values.to_numpy().reshape(-1, 3), + "conf": encoded_bbox[0]["conf"].values.to_numpy(), + "names": encoded_bbox[0]["names"].values.to_pylist(), +} +``` + +## License + +This project is licensed under Apache-2.0. Check out [NOTICE.md](../../NOTICE.md) for more information. diff --git a/node-hub/ultralytics-yolo/main.py b/node-hub/ultralytics-yolo/main.py new file mode 100644 index 00000000..9f4d9b15 --- /dev/null +++ b/node-hub/ultralytics-yolo/main.py @@ -0,0 +1,158 @@ +import os +import argparse + +import numpy as np +import pyarrow as pa + +from dora import Node +from ultralytics import YOLO + +LABELS = [ + "ABC", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +] + + +def main(): + # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables. + parser = argparse.ArgumentParser( + description="UltraLytics YOLO: This node is used to perform object detection using the UltraLytics YOLO model.") + + parser.add_argument("--name", type=str, required=False, help="The name of the node in the dataflow.", + default="ultralytics-yolo") + parser.add_argument("--model", type=str, required=False, + help="The name of the model file (e.g. yolov8n.pt).", default="yolov8n.pt") + + args = parser.parse_args() + + model_path = os.getenv("MODEL", args.model) + + model = YOLO(model_path) + node = Node(args.name) + + pa.array([]) # initialize pyarrow array + + for event in node: + event_type = event["type"] + + if event_type == "INPUT": + event_id = event["id"] + + if event_id == "image": + arrow_image = event["value"][0] + image = { + "width": np.uint32(arrow_image["width"].as_py()), + "height": np.uint32(arrow_image["height"].as_py()), + "channels": np.uint8(arrow_image["channels"].as_py()), + "data": arrow_image["data"].values.to_numpy().astype(np.uint8) + } + + frame = image["data"].reshape((image["height"], image["width"], 3)) + + frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) + results = model(frame, verbose=False) # includes NMS + + bboxes = np.array(results[0].boxes.xyxy.cpu()) + conf = np.array(results[0].boxes.conf.cpu()) + labels = np.array(results[0].boxes.cls.cpu()) + + names = [LABELS[int(label)] for label in labels] + + bbox = { + "bbox": bboxes.ravel(), + "conf": conf, + "names": names, + } + + node.send_output( + "bbox", + pa.array([bbox]), + event["metadata"], + ) + + elif event_type == "STOP": + break + elif event_type == "ERROR": + raise Exception(event["error"]) + + +if __name__ == "__main__": + main() diff --git a/node-hub/ultralytics-yolo/pyproject.toml b/node-hub/ultralytics-yolo/pyproject.toml new file mode 100644 index 00000000..0d7feacf --- /dev/null +++ b/node-hub/ultralytics-yolo/pyproject.toml @@ -0,0 +1,25 @@ +[tool.poetry] +name = "ultralytics-yolo" +version = "0.1" +authors = [ + "Haixuan Xavier Tao ", + "Enzo Le Van " +] +description = "Dora Node for object detection with Ultralytics YOLOv8" +readme = "README.md" + +packages = [ + { include = "main.py", to = "ultralytics_yolo" } +] + +[tool.poetry.dependencies] +dora-rs = "0.3.5" +numpy = "< 2.0.0" +ultralytics = "<= 8.2.52" + +[tool.poetry.scripts] +ultralytics-yolo = "ultralytics_yolo.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/nodes_hub/opencv-plot/plot.py b/nodes_hub/opencv-plot/plot.py deleted file mode 100644 index d2c4ce74..00000000 --- a/nodes_hub/opencv-plot/plot.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from dataclasses import dataclass - -import cv2 -import numpy as np - -from dora import Node - -CI = os.environ.get("CI") - -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) - -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -@dataclass -class Plotter: - frame: np.array = np.array([]) - bboxes: np.array = np.array([[]]) - conf: np.array = np.array([]) - label: np.array = np.array([]) - - -if __name__ == "__main__": - plotter = Plotter() - node = Node() - for event in node: - event_type = event["type"] - if event_type == "INPUT": - if event["id"] == "image": - frame = event["value"].to_numpy() - frame = frame.reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)).copy() - plotter.frame = frame - - elif event["id"] == "bbox": - bboxes = event["value"][0]["bbox"].values.to_numpy() - conf = event["value"][0]["conf"].values.to_numpy() - label = event["value"][0]["names"].values.to_pylist() - plotter.bboxes = np.reshape(bboxes, (-1, 4)) - plotter.conf = conf - plotter.label = label - continue - - for bbox in zip(plotter.bboxes, plotter.conf, plotter.label): - [ - [min_x, min_y, max_x, max_y], - confidence, - label, - ] = bbox - cv2.rectangle( - plotter.frame, - (int(min_x), int(min_y)), - (int(max_x), int(max_y)), - (0, 255, 0), - 2, - ) - - cv2.putText( - plotter.frame, - f"{label}, {confidence:0.2f}", - (int(max_x) - 120, int(max_y) - 10), - FONT, - 0.5, - (0, 255, 0), - 2, - 1, - ) - - if CI != "true": - cv2.imshow("frame", plotter.frame) - if cv2.waitKey(1) & 0xFF == ord("q"): - break diff --git a/nodes_hub/opencv-plot/requirements.txt b/nodes_hub/opencv-plot/requirements.txt deleted file mode 100644 index 7f3ef381..00000000 --- a/nodes_hub/opencv-plot/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -numpy<2.0.0 -opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/requirements.txt b/nodes_hub/opencv-video-capture/requirements.txt deleted file mode 100644 index 1db957a5..00000000 --- a/nodes_hub/opencv-video-capture/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy<2.0.0 -pyarrow -opencv-python \ No newline at end of file diff --git a/nodes_hub/opencv-video-capture/video_capture.py b/nodes_hub/opencv-video-capture/video_capture.py deleted file mode 100644 index cad4320c..00000000 --- a/nodes_hub/opencv-video-capture/video_capture.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import time - -import cv2 -import numpy as np -import pyarrow as pa - -from dora import Node - -CAM_INDEX = int(os.getenv("CAM_INDEX", "0")) -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) -MAX_DURATION = int(os.getenv("DURATION", "20")) -FONT = cv2.FONT_HERSHEY_SIMPLEX - - -start = time.time() - - -if __name__ == "__main__": - - video_capture = cv2.VideoCapture(CAM_INDEX) - node = Node() - - while time.time() - start < MAX_DURATION: - event = node.next() - if event is None: - break - if event is not None: - event_type = event["type"] - if event_type == "INPUT": - ret, frame = video_capture.read() - - # Fail to read camera - if not ret: - frame = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3), dtype=np.uint8) - cv2.putText( - frame, - "No Webcam was found at index %d" % (CAM_INDEX), - (int(30), int(30)), - FONT, - 0.75, - (255, 255, 255), - 2, - 1, - ) - - node.send_output( - "image", - pa.array(frame.ravel()), - event["metadata"], - ) diff --git a/nodes_hub/ultralytics-yolo/requirements.txt b/nodes_hub/ultralytics-yolo/requirements.txt deleted file mode 100644 index d8872ab6..00000000 --- a/nodes_hub/ultralytics-yolo/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy<2.0.0 -pyarrow -ultralytics \ No newline at end of file diff --git a/nodes_hub/ultralytics-yolo/yolo.py b/nodes_hub/ultralytics-yolo/yolo.py deleted file mode 100644 index 392c3ee9..00000000 --- a/nodes_hub/ultralytics-yolo/yolo.py +++ /dev/null @@ -1,50 +0,0 @@ -## Imports -import os - -import numpy as np -import pyarrow as pa -from ultralytics import YOLO - -from dora import Node - -## OS Environment variable -IMAGE_WIDTH = int(os.getenv("IMAGE_WIDTH", "640")) -IMAGE_HEIGHT = int(os.getenv("IMAGE_HEIGHT", "480")) -MODEL = os.getenv("MODEL", "yolov8n.pt") - -if __name__ == "__main__": - - model = YOLO(MODEL) - - node = Node("object_detection") - - for event in node: - event_type = event["type"] - if event_type == "INPUT": - event_id = event["id"] - if event_id == "image": - frame = ( - event["value"].to_numpy().reshape((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) - ) - frame = frame[:, :, ::-1] # OpenCV image (BGR to RGB) - results = model(frame, verbose=False) # includes NMS - # Process results - bboxes = np.array(results[0].boxes.xyxy.cpu()) - conf = np.array(results[0].boxes.conf.cpu()) - labels = np.array(results[0].boxes.cls.cpu()) - names = [model.names.get(label) for label in labels] - - node.send_output( - "bbox", - pa.array( - [ - { - "bbox": bboxes.ravel(), - "conf": conf, - "labels": labels, - "names": names, - } - ] - ), - event["metadata"], - )