From 75326ec0df4c0cde103411d1af53f70c43767842 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 16 Jul 2025 18:43:15 +0200 Subject: [PATCH] Document remaining fields --- libraries/message/src/descriptor.rs | 110 +++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 17 deletions(-) diff --git a/libraries/message/src/descriptor.rs b/libraries/message/src/descriptor.rs index e2f8e6e8..cbce13de 100644 --- a/libraries/message/src/descriptor.rs +++ b/libraries/message/src/descriptor.rs @@ -329,47 +329,123 @@ pub struct Node { #[serde(skip_serializing_if = "Option::is_none")] pub send_stdout_as: Option, - /// Git repository URL for remote nodes. + /// Build commands executed during `dora build`. Each line runs separately. + /// + /// The `build` key specifies the command that should be invoked for building the node. + /// The key expects a single- or multi-line string. + /// + /// Each line is run as a separate command. + /// Spaces are used to separate arguments. + /// + /// Note that all the environment variables specified in the [`env`](Self::env) field are also + /// applied to the build commands. + /// + /// ## Special treatment of `pip` + /// + /// Build lines that start with `pip` or `pip3` are treated in a special way: + /// If the `--uv` argument is passed to the `dora build` command, all `pip`/`pip3` commands are + /// run through the [`uv` package manager](https://docs.astral.sh/uv/). + /// + /// ## Example /// /// ```yaml - /// git: https://github.com/user/repo.git + /// nodes: + /// - id: build-example + /// build: cargo build -p receive_data --release + /// path: target/release/receive_data + /// - id: multi-line-example + /// build: | + /// pip install requirements.txt + /// pip install -e some/local/package + /// path: package /// ``` + /// + /// In the above example, the `pip` commands will be replaced by `uv pip` when run through + /// `dora build --uv`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub git: Option, + pub build: Option, - /// Git branch to checkout. Only one of branch/tag/rev allowed. + /// Git repository URL for downloading nodes. + /// + /// The `git` key allows downloading nodes (i.e. their source code) from git repositories. + /// This can be especially useful for distributed dataflows. + /// + /// When a `git` key is specified, `dora build` automatically clones the specified repository + /// (or reuse an existing clone). + /// Then it checks out the specified [`branch`](Self::branch), [`tag`](Self::tag), or + /// [`rev`](Self::rev), or the default branch if none of them are specified. + /// Afterwards it runs the [`build`](Self::build) command if specified. + /// + /// Note that the git clone directory is set as working directory for both the + /// [`build`](Self::build) command and the specified [`path`](Self::path). + /// + /// ## Example /// /// ```yaml - /// branch: main + /// nodes: + /// - id: rust-node + /// git: https://github.com/dora-rs/dora.git + /// build: cargo build -p rust-dataflow-example-node + /// path: target/debug/rust-dataflow-example-node /// ``` + /// + /// In the above example, `dora build` will first clone the specified `git` repository and then + /// run the specified `build` inside the local clone directory. + /// When `dora run` or `dora start` is invoked, the working directory will be the git clone + /// directory too. So a relative `path` will start from the clone directory. #[serde(default, skip_serializing_if = "Option::is_none")] - pub branch: Option, + pub git: Option, - /// Git tag to checkout. Only one of branch/tag/rev allowed. + /// Git branch to checkout after cloning. + /// + /// The `branch` field is only allowed in combination with the [`git`](#git) field. + /// It specifies the branch that should be checked out after cloning. + /// Only one of `branch`, `tag`, or `rev` can be specified. + /// + /// ## Example /// /// ```yaml - /// tag: v1.0.0 + /// nodes: + /// - id: rust-node + /// git: https://github.com/dora-rs/dora.git + /// branch: some-branch-name /// ``` #[serde(default, skip_serializing_if = "Option::is_none")] - pub tag: Option, + pub branch: Option, - /// Git commit hash to checkout. Only one of branch/tag/rev allowed. + /// Git tag to checkout after cloning. + /// + /// The `tag` field is only allowed in combination with the [`git`](#git) field. + /// It specifies the git tag that should be checked out after cloning. + /// Only one of `branch`, `tag`, or `rev` can be specified. + /// + /// ## Example /// /// ```yaml - /// rev: abc123def456 + /// nodes: + /// - id: rust-node + /// git: https://github.com/dora-rs/dora.git + /// tag: v0.3.0 /// ``` #[serde(default, skip_serializing_if = "Option::is_none")] - pub rev: Option, + pub tag: Option, - /// Build commands executed during `dora build`. Each line runs separately. + /// Git revision (e.g. commit hash) to checkout after cloning. + /// + /// The `rev` field is only allowed in combination with the [`git`](#git) field. + /// It specifies the git revision (e.g. a commit hash) that should be checked out after cloning. + /// Only one of `branch`, `tag`, or `rev` can be specified. + /// + /// ## Example /// /// ```yaml - /// build: | - /// pip install requirements.txt - /// cargo build --release + /// nodes: + /// - id: rust-node + /// git: https://github.com/dora-rs/dora.git + /// rev: 64ab0d7c /// ``` #[serde(default, skip_serializing_if = "Option::is_none")] - pub build: Option, + pub rev: Option, } #[derive(Debug, Clone, Serialize, Deserialize)]