diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b7882f9..e8c55ba3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,11 +310,18 @@ jobs: mv .venv/Scripts .venv/bin # venv is placed under `Scripts` on Windows fi source .venv/bin/activate - pip3 install maturin + pip3 install maturin black pylint pytest maturin build -m apis/python/node/Cargo.toml pip3 install target/wheels/* dora new test_python_project --lang python --internal-create-with-path-dependencies cd test_python_project + + # Check Compliancy + black . --check + pylint --disable=C,R **/*.py + pip install -e ./*/ + pytest + dora up dora list dora build dataflow.yml diff --git a/binaries/cli/src/lib.rs b/binaries/cli/src/lib.rs index 3bd2c1e6..a59e49cd 100644 --- a/binaries/cli/src/lib.rs +++ b/binaries/cli/src/lib.rs @@ -254,7 +254,7 @@ pub struct CommandNew { #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] enum Kind { Dataflow, - CustomNode, + Node, } #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] diff --git a/binaries/cli/src/template/c/mod.rs b/binaries/cli/src/template/c/mod.rs index 378c572c..879eeedb 100644 --- a/binaries/cli/src/template/c/mod.rs +++ b/binaries/cli/src/template/c/mod.rs @@ -18,7 +18,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()> } = args; match kind { - crate::Kind::CustomNode => create_custom_node(name, path, NODE), + crate::Kind::Node => create_custom_node(name, path, NODE), crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps), } } diff --git a/binaries/cli/src/template/cxx/mod.rs b/binaries/cli/src/template/cxx/mod.rs index 09935031..72b7769e 100644 --- a/binaries/cli/src/template/cxx/mod.rs +++ b/binaries/cli/src/template/cxx/mod.rs @@ -17,7 +17,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()> } = args; match kind { - crate::Kind::CustomNode => create_custom_node(name, path, NODE), + crate::Kind::Node => create_custom_node(name, path, NODE), crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps), } } diff --git a/binaries/cli/src/template/python/__node-name__/README.md b/binaries/cli/src/template/python/__node-name__/README.md new file mode 100644 index 00000000..81b12cc0 --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/README.md @@ -0,0 +1,37 @@ +# Node Name + +## Getting started + +- Install it with pip: + +```bash +pip install -e . +``` + +## Contribution Guide + +- Format with [black](https://github.com/psf/black): + +```bash +black . # Format +``` + +- Lint with [pylint](https://github.com/pylint-dev/pylint): + +```bash +pylint --disable=C,R --ignored-modules=cv2 . # Lint +``` + +- Test with [pytest](https://github.com/pytest-dev/pytest) + +```bash +pytest . # Test +``` + +## YAML Specification + +## Examples + +## License + +Node Name's code are released under the MIT License diff --git a/binaries/cli/src/template/python/__node-name__/__node_name__/__init__.py b/binaries/cli/src/template/python/__node-name__/__node_name__/__init__.py new file mode 100644 index 00000000..ac3cbef9 --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/__node_name__/__init__.py @@ -0,0 +1,11 @@ +import os + +# Define the path to the README file relative to the package directory +readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md") + +# Read the content of the README file +try: + with open(readme_path, "r", encoding="utf-8") as f: + __doc__ = f.read() +except FileNotFoundError: + __doc__ = "README file not found." diff --git a/binaries/cli/src/template/python/__node-name__/__node_name__/__main__.py b/binaries/cli/src/template/python/__node-name__/__node_name__/__main__.py new file mode 100644 index 00000000..bcbfde6d --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/__node_name__/__main__.py @@ -0,0 +1,5 @@ +from .main import main + + +if __name__ == "__main__": + main() diff --git a/binaries/cli/src/template/python/__node-name__/__node_name__/main.py b/binaries/cli/src/template/python/__node-name__/__node_name__/main.py new file mode 100644 index 00000000..823e2e09 --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/__node_name__/main.py @@ -0,0 +1,26 @@ +from dora import Node +import pyarrow as pa + + +def main(): + node = Node() + + for event in node: + if event["type"] == "INPUT": + if event["id"] == "TICK": + print( + f"""Node received: + id: {event["id"]}, + value: {event["value"]}, + metadata: {event["metadata"]}""" + ) + + elif event["id"] == "my_input_id": + # Warning: Make sure to add my_output_id and my_input_id within the dataflow. + node.send_output( + output_id="my_output_id", data=pa.array([1, 2, 3]), metadata={} + ) + + +if __name__ == "__main__": + main() diff --git a/binaries/cli/src/template/python/__node-name__/pyproject.toml b/binaries/cli/src/template/python/__node-name__/pyproject.toml new file mode 100644 index 00000000..a54fb19e --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/pyproject.toml @@ -0,0 +1,28 @@ +[tool.poetry] +name = "__node-name__" +version = "0.0.0" +authors = ["author"] +description = "Node Name" +license = "MIT License" +homepage = "https://github.com/dora-rs/dora.git" +documentation = "https://github.com/dora-rs/dora/blob/main/node-hub/__node-name__/README.md" +readme = "README.md" +packages = [{ include = "__node_name__" }] + +[tool.poetry.dependencies] +dora-rs = "^0.3.6" +numpy = "< 2.0.0" +pyarrow = ">= 5.0.0" +python = "^3.7" + +[tool.poetry.dev-dependencies] +pytest = ">= 8.3.4" +pylint = ">= 3.3.2" +black = ">= 24.10" + +[tool.poetry.scripts] +__node-name__ = "__node_name__.main:main" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" diff --git a/binaries/cli/src/template/python/__node-name__/tests/test___node_name__.py b/binaries/cli/src/template/python/__node-name__/tests/test___node_name__.py new file mode 100644 index 00000000..7fefda95 --- /dev/null +++ b/binaries/cli/src/template/python/__node-name__/tests/test___node_name__.py @@ -0,0 +1,9 @@ +import pytest + + +def test_import_main(): + from __node_name__.main import main + + # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow. + with pytest.raises(RuntimeError): + main() diff --git a/binaries/cli/src/template/python/dataflow-template.yml b/binaries/cli/src/template/python/dataflow-template.yml index 2fa63651..3797b98c 100644 --- a/binaries/cli/src/template/python/dataflow-template.yml +++ b/binaries/cli/src/template/python/dataflow-template.yml @@ -1,19 +1,20 @@ nodes: - id: talker_1 - path: talker_1/talker_1.py + path: talker-1/talker_1/main.py inputs: tick: dora/timer/millis/100 outputs: - speech + - id: talker_2 - path: talker_2/talker_2.py + path: talker-2/talker_2/main.py inputs: tick: dora/timer/secs/2 outputs: - speech - id: listener_1 - path: listener_1/listener_1.py + path: listener-1/listener_1/main.py inputs: speech-1: talker_1/speech speech-2: talker_2/speech diff --git a/binaries/cli/src/template/python/listener/listener-template.py b/binaries/cli/src/template/python/listener/listener-template.py index 82c8f809..b94fa167 100644 --- a/binaries/cli/src/template/python/listener/listener-template.py +++ b/binaries/cli/src/template/python/listener/listener-template.py @@ -1,11 +1,13 @@ from dora import Node -import pyarrow as pa -node = Node() -for event in node: - if event["type"] == "INPUT": - message = event["value"][0].as_py() - print( - f"""I heard {message} from {event["id"]}""" - ) +def main(): + node = Node() + for event in node: + if event["type"] == "INPUT": + message = event["value"][0].as_py() + print(f"""I heard {message} from {event["id"]}""") + + +if __name__ == "__main__": + main() diff --git a/binaries/cli/src/template/python/mod.rs b/binaries/cli/src/template/python/mod.rs index 4c8eb435..911141eb 100644 --- a/binaries/cli/src/template/python/mod.rs +++ b/binaries/cli/src/template/python/mod.rs @@ -4,7 +4,13 @@ use std::{ path::{Path, PathBuf}, }; -const NODE_PY: &str = include_str!("node/node-template.py"); +const MAIN_PY: &str = include_str!("__node-name__/__node_name__/main.py"); +const _MAIN_PY: &str = include_str!("__node-name__/__node_name__/__main__.py"); +const _INIT_PY: &str = include_str!("__node-name__/__node_name__/__init__.py"); +const _TEST_PY: &str = include_str!("__node-name__/tests/test___node_name__.py"); +const PYPROJECT_TOML: &str = include_str!("__node-name__/pyproject.toml"); +const README_MD: &str = include_str!("__node-name__/README.md"); + const TALKER_PY: &str = include_str!("talker/talker-template.py"); const LISTENER_PY: &str = include_str!("listener/listener-template.py"); @@ -17,29 +23,76 @@ pub fn create(args: crate::CommandNew) -> eyre::Result<()> { } = args; match kind { - crate::Kind::CustomNode => create_custom_node(name, path, NODE_PY), + crate::Kind::Node => create_custom_node(name, path, MAIN_PY), crate::Kind::Dataflow => create_dataflow(name, path), } } +fn replace_space(file: &str, name: &str) -> String { + let mut file = file.replace("__node-name__", &name.replace(" ", "-")); + file = file.replace("__node_name__", &name.replace("-", "_").replace(" ", "_")); + file.replace("Node Name", &name) +} fn create_custom_node( name: String, path: Option, - template_scripts: &str, + main: &str, ) -> Result<(), eyre::ErrReport> { // create directories - let root = path.as_deref().unwrap_or_else(|| Path::new(&name)); - fs::create_dir(root) - .with_context(|| format!("failed to create directory `{}`", root.display()))?; + let root = path.unwrap_or_else(|| PathBuf::from(name.replace(" ", "-"))); + let module_path = root.join(name.replace(" ", "_").replace("-", "_")); + fs::create_dir(&root) + .with_context(|| format!("failed to create root directory `{}`", &root.display()))?; + + fs::create_dir(&module_path) + .with_context(|| format!("failed to create module directory `{}`", &root.display()))?; - let node_path = root.join(format!("{name}.py")); - fs::write(&node_path, template_scripts) + fs::create_dir(&root.join("tests")) + .with_context(|| format!("failed to create tests directory `{}`", &root.display()))?; + + // PYPROJECT.toml + let node_path = root.join("pyproject.toml"); + let pyproject = replace_space(PYPROJECT_TOML, &name); + fs::write(&node_path, pyproject) + .with_context(|| format!("failed to write `{}`", node_path.display()))?; + + // README.md + let node_path = root.join("README.md"); + fs::write(&node_path, README_MD.replace("Node Name", &name)) + .with_context(|| format!("failed to write `{}`", node_path.display()))?; + + // main.py + let node_path = module_path.join("main.py"); + fs::write(&node_path, main) + .with_context(|| format!("failed to write `{}`", node_path.display()))?; + + // __main__.py + let node_path = module_path.join("__main__.py"); + fs::write(&node_path, _MAIN_PY) + .with_context(|| format!("failed to write `{}`", node_path.display()))?; + + // __init__.py + let node_path = module_path.join("__init__.py"); + fs::write(&node_path, _INIT_PY) + .with_context(|| format!("failed to write `{}`", node_path.display()))?; + + // tests/tests___node_name__.py + let node_path = root + .join("tests") + .join(format!("test_{}.py", name.replace(" ", "_"))); + let file = replace_space(_TEST_PY, &name); + fs::write(&node_path, file) .with_context(|| format!("failed to write `{}`", node_path.display()))?; println!( "Created new Python node `{name}` at {}", - Path::new(".").join(root).display() + Path::new(".").join(&root).display() ); + println!(" cd {}", Path::new(".").join(&root).display()); + println!(" pip install -e . # Install",); + println!(" black . # Format"); + println!(" pylint --disable=C,R . # Lint",); + println!(" pytest . # Test"); Ok(()) } @@ -57,18 +110,18 @@ fn create_dataflow(name: String, path: Option) -> Result<(), eyre::ErrR // create directories let root = path.as_deref().unwrap_or_else(|| Path::new(&name)); fs::create_dir(root) - .with_context(|| format!("failed to create directory `{}`", root.display()))?; + .with_context(|| format!("failed to create module directory `{}`", root.display()))?; let dataflow_yml = DATAFLOW_YML.replace("___name___", &name); let dataflow_yml_path = root.join("dataflow.yml"); fs::write(&dataflow_yml_path, dataflow_yml) .with_context(|| format!("failed to write `{}`", dataflow_yml_path.display()))?; - create_custom_node("talker_1".into(), Some(root.join("talker_1")), TALKER_PY)?; - create_custom_node("talker_2".into(), Some(root.join("talker_2")), TALKER_PY)?; + create_custom_node("talker 1".into(), Some(root.join("talker-1")), TALKER_PY)?; + create_custom_node("talker 2".into(), Some(root.join("talker-2")), TALKER_PY)?; create_custom_node( - "listener_1".into(), - Some(root.join("listener_1")), + "listener 1".into(), + Some(root.join("listener-1")), LISTENER_PY, )?; diff --git a/binaries/cli/src/template/python/node/node-template.py b/binaries/cli/src/template/python/node/node-template.py deleted file mode 100644 index f29c0a99..00000000 --- a/binaries/cli/src/template/python/node/node-template.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from dora import Node - -node = Node() - -event = node.next() -if event["type"] == "INPUT": - print( - f"""Node received: - id: {event["id"]}, - value: {event["value"]}, - metadata: {event["metadata"]}""" - ) diff --git a/binaries/cli/src/template/python/talker/talker-template.py b/binaries/cli/src/template/python/talker/talker-template.py index 93324e40..7cc5456a 100644 --- a/binaries/cli/src/template/python/talker/talker-template.py +++ b/binaries/cli/src/template/python/talker/talker-template.py @@ -1,14 +1,20 @@ from dora import Node import pyarrow as pa -node = Node() - -for event in node: - if event["type"] == "INPUT": - print( - f"""Node received: - id: {event["id"]}, - value: {event["value"]}, - metadata: {event["metadata"]}""" - ) - node.send_output("speech", pa.array(["Hello World"])) + +def main(): + node = Node() + + for event in node: + if event["type"] == "INPUT": + print( + f"""Node received: + id: {event["id"]}, + value: {event["value"]}, + metadata: {event["metadata"]}""" + ) + node.send_output("speech", pa.array(["Hello World"])) + + +if __name__ == "__main__": + main() diff --git a/binaries/cli/src/template/rust/mod.rs b/binaries/cli/src/template/rust/mod.rs index a0f4717c..f2c9fd9c 100644 --- a/binaries/cli/src/template/rust/mod.rs +++ b/binaries/cli/src/template/rust/mod.rs @@ -18,7 +18,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()> } = args; match kind { - crate::Kind::CustomNode => create_custom_node(name, path, use_path_deps, MAIN_RS), + crate::Kind::Node => create_custom_node(name, path, use_path_deps, MAIN_RS), crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps), } }