Rewrite python template to make them pip installable (#744)

This template makes that new python nodes using the template are easily
pip compliant as well as compliant with our node-hub.

It makes it possible to create node faster without having to copy/paste
old node.

.github/workflows/ci.yml

@@ -310,11 +310,18 @@ jobs:
             mv .venv/Scripts .venv/bin    # venv is placed under `Scripts` on Windows
           fi
           source .venv/bin/activate
-          pip3 install maturin
+          pip3 install maturin black pylint pytest
           maturin build -m apis/python/node/Cargo.toml
           pip3 install target/wheels/*
           dora new test_python_project --lang python --internal-create-with-path-dependencies
           cd test_python_project
+
+          # Check Compliancy
+          black . --check
+          pylint --disable=C,R **/*.py
+          pip install -e ./*/
+          pytest
+
           dora up
           dora list
           dora build dataflow.yml

binaries/cli/src/lib.rs

@@ -254,7 +254,7 @@ pub struct CommandNew {
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
 enum Kind {
     Dataflow,
-    CustomNode,
+    Node,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]

binaries/cli/src/template/c/mod.rs

@@ -18,7 +18,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()>
     } = args;
 
     match kind {
-        crate::Kind::CustomNode => create_custom_node(name, path, NODE),
+        crate::Kind::Node => create_custom_node(name, path, NODE),
         crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps),
     }
 }

binaries/cli/src/template/cxx/mod.rs

@@ -17,7 +17,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()>
     } = args;
 
     match kind {
-        crate::Kind::CustomNode => create_custom_node(name, path, NODE),
+        crate::Kind::Node => create_custom_node(name, path, NODE),
         crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps),
     }
 }

binaries/cli/src/template/python/__node-name__/README.md

@@ -0,0 +1,37 @@
+# Node Name
+
+## Getting started
+
+- Install it with pip:
+
+```bash
+pip install -e .
+```
+
+## Contribution Guide
+
+- Format with [black](https://github.com/psf/black):
+
+```bash
+black . # Format
+```
+
+- Lint with [pylint](https://github.com/pylint-dev/pylint):
+
+```bash
+pylint --disable=C,R --ignored-modules=cv2 . # Lint
+```
+
+- Test with [pytest](https://github.com/pytest-dev/pytest)
+
+```bash
+pytest . # Test
+```
+
+## YAML Specification
+
+## Examples
+
+## License
+
+Node Name's code are released under the MIT License

binaries/cli/src/template/python/__node-name__/__node_name__/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+    with open(readme_path, "r", encoding="utf-8") as f:
+        __doc__ = f.read()
+except FileNotFoundError:
+    __doc__ = "README file not found."

binaries/cli/src/template/python/__node-name__/__node_name__/__main__.py

@@ -0,0 +1,5 @@
+from .main import main
+
+
+if __name__ == "__main__":
+    main()

binaries/cli/src/template/python/__node-name__/__node_name__/main.py

@@ -0,0 +1,26 @@
+from dora import Node
+import pyarrow as pa
+
+
+def main():
+    node = Node()
+
+    for event in node:
+        if event["type"] == "INPUT":
+            if event["id"] == "TICK":
+                print(
+                    f"""Node received:
+                id: {event["id"]},
+                value: {event["value"]},
+                metadata: {event["metadata"]}"""
+                )
+
+            elif event["id"] == "my_input_id":
+                # Warning: Make sure to add my_output_id and my_input_id within the dataflow.
+                node.send_output(
+                    output_id="my_output_id", data=pa.array([1, 2, 3]), metadata={}
+                )
+
+
+if __name__ == "__main__":
+    main()

binaries/cli/src/template/python/__node-name__/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "__node-name__"
+version = "0.0.0"
+authors = ["author"]
+description = "Node Name"
+license = "MIT License"
+homepage = "https://github.com/dora-rs/dora.git"
+documentation = "https://github.com/dora-rs/dora/blob/main/node-hub/__node-name__/README.md"
+readme = "README.md"
+packages = [{ include = "__node_name__" }]
+
+[tool.poetry.dependencies]
+dora-rs = "^0.3.6"
+numpy = "< 2.0.0"
+pyarrow = ">= 5.0.0"
+python = "^3.7"
+
+[tool.poetry.dev-dependencies]
+pytest = ">= 8.3.4"
+pylint = ">= 3.3.2"
+black = ">= 24.10"
+
+[tool.poetry.scripts]
+__node-name__ = "__node_name__.main:main"
+
+[build-system]
+requires = ["poetry-core>=1.8.0"]
+build-backend = "poetry.core.masonry.api"

binaries/cli/src/template/python/__node-name__/tests/test___node_name__.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+def test_import_main():
+    from __node_name__.main import main
+
+    # Check that everything is working, and catch dora Runtime Exception as we're not running in a dora dataflow.
+    with pytest.raises(RuntimeError):
+        main()

binaries/cli/src/template/python/dataflow-template.yml

@@ -1,19 +1,20 @@
 nodes:
   - id: talker_1
-    path: talker_1/talker_1.py
+    path: talker-1/talker_1/main.py
     inputs:
       tick: dora/timer/millis/100
     outputs:
       - speech
+
   - id: talker_2
-    path: talker_2/talker_2.py
+    path: talker-2/talker_2/main.py
     inputs:
       tick: dora/timer/secs/2
     outputs:
       - speech
 
   - id: listener_1
-    path: listener_1/listener_1.py
+    path: listener-1/listener_1/main.py
     inputs:
       speech-1: talker_1/speech
       speech-2: talker_2/speech

binaries/cli/src/template/python/listener/listener-template.py

@@ -1,11 +1,13 @@
 from dora import Node
-import pyarrow as pa
 
-node = Node()
 
-for event in node:
-    if event["type"] == "INPUT":
-        message = event["value"][0].as_py()
-        print(
-            f"""I heard {message} from {event["id"]}"""
-        )
+def main():
+    node = Node()
+    for event in node:
+        if event["type"] == "INPUT":
+            message = event["value"][0].as_py()
+            print(f"""I heard {message} from {event["id"]}""")
+
+
+if __name__ == "__main__":
+    main()

binaries/cli/src/template/python/mod.rs

@@ -4,7 +4,13 @@ use std::{
     path::{Path, PathBuf},
 };
 
-const NODE_PY: &str = include_str!("node/node-template.py");
+const MAIN_PY: &str = include_str!("__node-name__/__node_name__/main.py");
+const _MAIN_PY: &str = include_str!("__node-name__/__node_name__/__main__.py");
+const _INIT_PY: &str = include_str!("__node-name__/__node_name__/__init__.py");
+const _TEST_PY: &str = include_str!("__node-name__/tests/test___node_name__.py");
+const PYPROJECT_TOML: &str = include_str!("__node-name__/pyproject.toml");
+const README_MD: &str = include_str!("__node-name__/README.md");
+
 const TALKER_PY: &str = include_str!("talker/talker-template.py");
 const LISTENER_PY: &str = include_str!("listener/listener-template.py");
 
@@ -17,29 +23,76 @@ pub fn create(args: crate::CommandNew) -> eyre::Result<()> {
     } = args;
 
     match kind {
-        crate::Kind::CustomNode => create_custom_node(name, path, NODE_PY),
+        crate::Kind::Node => create_custom_node(name, path, MAIN_PY),
         crate::Kind::Dataflow => create_dataflow(name, path),
     }
 }
 
+fn replace_space(file: &str, name: &str) -> String {
+    let mut file = file.replace("__node-name__", &name.replace(" ", "-"));
+    file = file.replace("__node_name__", &name.replace("-", "_").replace(" ", "_"));
+    file.replace("Node Name", &name)
+}
 fn create_custom_node(
     name: String,
     path: Option<PathBuf>,
-    template_scripts: &str,
+    main: &str,
 ) -> Result<(), eyre::ErrReport> {
     // create directories
-    let root = path.as_deref().unwrap_or_else(|| Path::new(&name));
-    fs::create_dir(root)
-        .with_context(|| format!("failed to create directory `{}`", root.display()))?;
+    let root = path.unwrap_or_else(|| PathBuf::from(name.replace(" ", "-")));
+    let module_path = root.join(name.replace(" ", "_").replace("-", "_"));
+    fs::create_dir(&root)
+        .with_context(|| format!("failed to create root directory `{}`", &root.display()))?;
+
+    fs::create_dir(&module_path)
+        .with_context(|| format!("failed to create module directory `{}`", &root.display()))?;
 
-    let node_path = root.join(format!("{name}.py"));
-    fs::write(&node_path, template_scripts)
+    fs::create_dir(&root.join("tests"))
+        .with_context(|| format!("failed to create tests directory `{}`", &root.display()))?;
+
+    // PYPROJECT.toml
+    let node_path = root.join("pyproject.toml");
+    let pyproject = replace_space(PYPROJECT_TOML, &name);
+    fs::write(&node_path, pyproject)
+        .with_context(|| format!("failed to write `{}`", node_path.display()))?;
+
+    // README.md
+    let node_path = root.join("README.md");
+    fs::write(&node_path, README_MD.replace("Node Name", &name))
+        .with_context(|| format!("failed to write `{}`", node_path.display()))?;
+
+    // main.py
+    let node_path = module_path.join("main.py");
+    fs::write(&node_path, main)
+        .with_context(|| format!("failed to write `{}`", node_path.display()))?;
+
+    // __main__.py
+    let node_path = module_path.join("__main__.py");
+    fs::write(&node_path, _MAIN_PY)
+        .with_context(|| format!("failed to write `{}`", node_path.display()))?;
+
+    // __init__.py
+    let node_path = module_path.join("__init__.py");
+    fs::write(&node_path, _INIT_PY)
+        .with_context(|| format!("failed to write `{}`", node_path.display()))?;
+
+    // tests/tests___node_name__.py
+    let node_path = root
+        .join("tests")
+        .join(format!("test_{}.py", name.replace(" ", "_")));
+    let file = replace_space(_TEST_PY, &name);
+    fs::write(&node_path, file)
         .with_context(|| format!("failed to write `{}`", node_path.display()))?;
 
     println!(
         "Created new Python node `{name}` at {}",
-        Path::new(".").join(root).display()
+        Path::new(".").join(&root).display()
     );
+    println!("   cd {}", Path::new(".").join(&root).display());
+    println!("   pip install -e . # Install",);
+    println!("   black . # Format");
+    println!("   pylint  --disable=C,R . # Lint",);
+    println!("   pytest . # Test");
 
     Ok(())
 }
@@ -57,18 +110,18 @@ fn create_dataflow(name: String, path: Option<PathBuf>) -> Result<(), eyre::ErrR
     // create directories
     let root = path.as_deref().unwrap_or_else(|| Path::new(&name));
     fs::create_dir(root)
-        .with_context(|| format!("failed to create directory `{}`", root.display()))?;
+        .with_context(|| format!("failed to create module directory `{}`", root.display()))?;
 
     let dataflow_yml = DATAFLOW_YML.replace("___name___", &name);
     let dataflow_yml_path = root.join("dataflow.yml");
     fs::write(&dataflow_yml_path, dataflow_yml)
         .with_context(|| format!("failed to write `{}`", dataflow_yml_path.display()))?;
 
-    create_custom_node("talker_1".into(), Some(root.join("talker_1")), TALKER_PY)?;
-    create_custom_node("talker_2".into(), Some(root.join("talker_2")), TALKER_PY)?;
+    create_custom_node("talker 1".into(), Some(root.join("talker-1")), TALKER_PY)?;
+    create_custom_node("talker 2".into(), Some(root.join("talker-2")), TALKER_PY)?;
     create_custom_node(
-        "listener_1".into(),
-        Some(root.join("listener_1")),
+        "listener 1".into(),
+        Some(root.join("listener-1")),
         LISTENER_PY,
     )?;
 

binaries/cli/src/template/python/node/node-template.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-from dora import Node
-
-node = Node()
-
-event = node.next()
-if event["type"] == "INPUT":
-    print(
-        f"""Node received:
-    id: {event["id"]},
-    value: {event["value"]},
-    metadata: {event["metadata"]}"""
-    )

binaries/cli/src/template/python/talker/talker-template.py

@@ -1,14 +1,20 @@
 from dora import Node
 import pyarrow as pa
 
-node = Node()
-
-for event in node:
-    if event["type"] == "INPUT":
-        print(
-            f"""Node received:
-        id: {event["id"]},
-        value: {event["value"]},
-        metadata: {event["metadata"]}"""
-        )
-        node.send_output("speech", pa.array(["Hello World"]))
+
+def main():
+    node = Node()
+
+    for event in node:
+        if event["type"] == "INPUT":
+            print(
+                f"""Node received:
+            id: {event["id"]},
+            value: {event["value"]},
+            metadata: {event["metadata"]}"""
+            )
+            node.send_output("speech", pa.array(["Hello World"]))
+
+
+if __name__ == "__main__":
+    main()

binaries/cli/src/template/rust/mod.rs

@@ -18,7 +18,7 @@ pub fn create(args: crate::CommandNew, use_path_deps: bool) -> eyre::Result<()>
     } = args;
 
     match kind {
-        crate::Kind::CustomNode => create_custom_node(name, path, use_path_deps, MAIN_RS),
+        crate::Kind::Node => create_custom_node(name, path, use_path_deps, MAIN_RS),
         crate::Kind::Dataflow => create_dataflow(name, path, use_path_deps),
     }
 }