From 9fe69ccd6703aeea5c1e6d8e3e4712c4d5c7b36a Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Aug 2022 16:08:01 +0200 Subject: [PATCH 1/8] Redesign of the `docs` as an `mdBook` This initial commit moves many of the documentation into a `mdBook` format version for ease of readibility. It includes a github action workflows that publish the book on the default `gh-pages` branch to be publish on github page. --- .github/workflows/gh-pages.yml | 32 ++++++++++++++++++++++++ README.md | 2 +- docs/.gitignore | 1 + docs/book.toml | 6 +++++ docs/design/README.md | 7 ------ docs/src/SUMMARY.md | 22 ++++++++++++++++ docs/src/communication-layer.md | 30 ++++++++++++++++++++++ docs/{design => src}/dataflow-config.md | 0 docs/src/getting-started.md | 17 +++++++++++++ docs/src/installation.md | 19 ++++++++++++++ docs/src/introduction.md | 30 ++++++++++++++++++++++ docs/{design => src}/logo.svg | 0 docs/{design => src}/overview.md | 0 docs/{design => src}/overview.svg | 0 docs/{design => src}/rust-client.md | 10 ++------ docs/{design => src}/state-management.md | 0 16 files changed, 160 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/gh-pages.yml create mode 100644 docs/.gitignore create mode 100644 docs/book.toml delete mode 100644 docs/design/README.md create mode 100644 docs/src/SUMMARY.md create mode 100644 docs/src/communication-layer.md rename docs/{design => src}/dataflow-config.md (100%) create mode 100644 docs/src/getting-started.md create mode 100644 docs/src/installation.md create mode 100644 docs/src/introduction.md rename docs/{design => src}/logo.svg (100%) rename docs/{design => src}/overview.md (100%) rename docs/{design => src}/overview.svg (100%) rename docs/{design => src}/rust-client.md (93%) rename docs/{design => src}/state-management.md (100%) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 00000000..7a4417f1 --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,32 @@ + +name: github pages + +on: + push: + branches: + - main + pull_request: + + +jobs: + deploy: + runs-on: ubuntu-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + steps: + - uses: actions/checkout@v2 + + - name: Setup mdBook + uses: peaceiris/actions-mdbook@v1 + with: + mdbook-version: '0.4.10' + # mdbook-version: 'latest' + + - run: mdbook build + + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/main' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/book \ No newline at end of file diff --git a/README.md b/README.md index bfbbccb3..d1b80dc1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..4e42a1bc --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +book/ \ No newline at end of file diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 00000000..e9cca438 --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,6 @@ +[book] +authors = ["Haixuan Xavier Tao", "Philipp Oppermann"] +language = "en" +multilingual = false +src = "src" +title = "dora-rs" diff --git a/docs/design/README.md b/docs/design/README.md deleted file mode 100644 index 70cf1e00..00000000 --- a/docs/design/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Design - -This folder contains our work-in-progress design documents: - -- [**Overview:**](overview.md) A high level overview of dora's different components and the differences between operators and custom nodes. -- [**Dataflow Configuration:**](dataflow-config.md) Specification of dora's YAML-based dataflow configuration format. -- [**State Management:**](state-management.md) Describes dora's state management capabilities for state sharing and recovery. diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md new file mode 100644 index 00000000..2d3fa62a --- /dev/null +++ b/docs/src/SUMMARY.md @@ -0,0 +1,22 @@ +# Summary + +[Introduction](./introduction.md) + +--- + +# User Guide + +- [installation](./installation.md) +- [Getting started](./getting-started.md) + +# Reference Guide + + +- [Overview](overview.md) A high level overview of dora's different components and the differences between operators and custom nodes. +- [Dataflow Configuration](dataflow-config.md) Specification of dora's YAML-based dataflow configuration format. + +# Brainstorming Ideas + +- [State Management](state-management.md) Describes dora's state management capabilities for state sharing and recovery. +- [Librairy vs Framework](rust-client.md) +- [Middleware Layer Abstraction](./communication-layer.md) \ No newline at end of file diff --git a/docs/src/communication-layer.md b/docs/src/communication-layer.md new file mode 100644 index 00000000..593822f0 --- /dev/null +++ b/docs/src/communication-layer.md @@ -0,0 +1,30 @@ +# Middleware (communication) layer abstraction (MLA) + +`dora` needs to implement MLA as a separate crate to provides a middleware abstraction layer that enables scalable, high performance communications for inter async tasks, intra-process (OS threads), interprocess communication on a single computer node or between different nodes in a computer network. MLA needs to support different communication patterns: +- publish-subscribe push / push pattern - the published message is pushed to subscribers +- publish-subscribe push / pull pattern - the published message is write to storage and later pulled by subscribers +- Request / reply pattern +- Point-to-point pattern +- Client / Server pattern + +The MLA needs to abstract following details: + +- inter-async tasks (e.g., tokio channels), intraprocess (OS threads, e.g., shared memory), interprocess and inter-host / inter-network communication +- different transport layer implementations (shared memory, UDP, TCP) +- builtin support for multiple serialization / deserialization protocols, e.g, capnproto, protobuf, flatbuffers etc +- different language bindings to Rust, Python, C, C++ etc +- telemetry tools for logs, metrics, distributed tracing, live data monitoring (e.g., tap a live data), recording and replay + +Rust eco-system has abundant crates to provide underlaying communications, e.g.,: +- tokio / crossbeam provides different types of channels serving different purpose: mpsc, oneshot, broadcast, watch etc +- Tonic provides gRPC services +- Tower provides request/reply service +- Zenoh middleware provides many different pub/sub capabilities + +MLA also needs to provide high level APIs: +- publish(topic, value, optional fields):- optional fields may contain senders' identify to help MLA logics to satify above requirements +- subscriber(topic, optional fields)-> future streams +- put(key, value, optional fields) +- get(key, optional fields) -> value +- send(key, msg, optional fields) +- recv(key, optional fields)->value diff --git a/docs/design/dataflow-config.md b/docs/src/dataflow-config.md similarity index 100% rename from docs/design/dataflow-config.md rename to docs/src/dataflow-config.md diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md new file mode 100644 index 00000000..7d561161 --- /dev/null +++ b/docs/src/getting-started.md @@ -0,0 +1,17 @@ +### 2. Compile the Rust example operator: + +```bash +cargo build --manifest-path ../examples/example-operator/Cargo.toml --release +``` + +### 3. Compile the C example operator: +```bash +cd ../../examples/c-operator +cp ../../apis/c/operator/api.h . +clang -c operator.c +clang -shared -v operator.o -o operator.so +``` +- Run the `mini-dataflow` example using `cargo run --release -- run examples/mini-dataflow.yml` + - This spawns a `timer` source, which sends the current time periodically, and a `logger` sink, which prints the incoming data. + - The `timer` will exit after 100 iterations. The other nodes/operators will then exit as well because all sources closed. + diff --git a/docs/src/installation.md b/docs/src/installation.md new file mode 100644 index 00000000..8a28f502 --- /dev/null +++ b/docs/src/installation.md @@ -0,0 +1,19 @@ +# Installation + +This project is in early development, and many features have yet to be implemented with breaking changes. Please don't take for granted the current design. The installation process will be streamlined in the future. + +### 1. Compile the dora-coordinator + +The `dora-coordinator` is responsible for reading the dataflow descriptor file and launching the operators accordingly. + +Build it using: +```bash +cargo build -p dora-coordinator --examples --release +``` + +### 2. Compile the dora-runtime for operators + +The `dora-runtime` is responsible for managing a set of operators. +```bash +cargo build -p dora-runtime --release +``` diff --git a/docs/src/introduction.md b/docs/src/introduction.md new file mode 100644 index 00000000..93a33dc4 --- /dev/null +++ b/docs/src/introduction.md @@ -0,0 +1,30 @@ +# Welcome to `dora`! + +`dora` goal is to be a low latency, composable, and distributed data flow. + +By using `dora`, you can define robotic applications as a graph of nodes that can be easily swapped and replaced. Those nodes can be shared and implemented in different languages such as Rust, Python or C. `dora` will then connect those nodes and try to provide as many features as possible to facilitate the dataflow. + +## ✨ Features that we want to provide + +Composability as: +- [x] `YAML` declarative programming +- [ ] language-agnostic: + - [x] Rust + - [x] C + - [ ] Python +- [ ] Isolated operator and node that can be reused. + +Low latency as: +- [x] written in ...Cough...blazingly fast ...Cough... Rust. +- [ ] Minimal abstraction close to the metal. + +Distributed as: +- [x] PubSub communication with [`zenoh`](https://github.com/eclipse-zenoh/zenoh) +- [x] Distributed telemetry with [`opentelemetry`](https://github.com/open-telemetry/opentelemetry-rust) + + + +## ⚖️ LICENSE + +This project is licensed under Apache-2.0. Check out [NOTICE.md](NOTICE.md) for more information. + diff --git a/docs/design/logo.svg b/docs/src/logo.svg similarity index 100% rename from docs/design/logo.svg rename to docs/src/logo.svg diff --git a/docs/design/overview.md b/docs/src/overview.md similarity index 100% rename from docs/design/overview.md rename to docs/src/overview.md diff --git a/docs/design/overview.svg b/docs/src/overview.svg similarity index 100% rename from docs/design/overview.svg rename to docs/src/overview.svg diff --git a/docs/design/rust-client.md b/docs/src/rust-client.md similarity index 93% rename from docs/design/rust-client.md rename to docs/src/rust-client.md index ee9b0066..d853e6d3 100644 --- a/docs/design/rust-client.md +++ b/docs/src/rust-client.md @@ -1,10 +1,4 @@ -# Rust Client Design - -## Brainstorm - -framework vs library - -### Framework +# Framework - Runtime process - Talks with other runtime processes @@ -27,7 +21,7 @@ framework vs library - by runtime -> aggregation specified in config file - by operator -> custom handling possible -### Library +# Library - All sources/operator/sinks are separate processes that link a runtime library - "Orchestrator" process diff --git a/docs/design/state-management.md b/docs/src/state-management.md similarity index 100% rename from docs/design/state-management.md rename to docs/src/state-management.md From cbc49db4fe286ea75780ef18b6fdc38235eef051 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Mon, 8 Aug 2022 16:24:28 +0200 Subject: [PATCH 2/8] fix github workflow --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 7a4417f1..ed7f0a0b 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -22,7 +22,7 @@ jobs: mdbook-version: '0.4.10' # mdbook-version: 'latest' - - run: mdbook build + - run: mdbook build docs - name: Deploy uses: peaceiris/actions-gh-pages@v3 From 1bf59f67d626cee68842052412b2a9b14a25eb6f Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Tue, 9 Aug 2022 22:23:42 +0200 Subject: [PATCH 3/8] fix documentation --- docs/src/communication-layer.md | 2 ++ docs/src/introduction.md | 8 ++++---- docs/src/{rust-client.md => library-vs-framework.md} | 0 3 files changed, 6 insertions(+), 4 deletions(-) rename docs/src/{rust-client.md => library-vs-framework.md} (100%) diff --git a/docs/src/communication-layer.md b/docs/src/communication-layer.md index 593822f0..bf06f231 100644 --- a/docs/src/communication-layer.md +++ b/docs/src/communication-layer.md @@ -28,3 +28,5 @@ MLA also needs to provide high level APIs: - get(key, optional fields) -> value - send(key, msg, optional fields) - recv(key, optional fields)->value + +More info here: https://github.com/dora-rs/dora/discussions/53 \ No newline at end of file diff --git a/docs/src/introduction.md b/docs/src/introduction.md index 93a33dc4..ee9a1c2c 100644 --- a/docs/src/introduction.md +++ b/docs/src/introduction.md @@ -8,15 +8,15 @@ By using `dora`, you can define robotic applications as a graph of nodes that ca Composability as: - [x] `YAML` declarative programming -- [ ] language-agnostic: +- [x] language-agnostic: - [x] Rust - [x] C - - [ ] Python -- [ ] Isolated operator and node that can be reused. + - [x] Python +- [ ] Isolated operators and nodes that can be reused. Low latency as: - [x] written in ...Cough...blazingly fast ...Cough... Rust. -- [ ] Minimal abstraction close to the metal. +- [ ] Minimal abstraction, close to the metal. Distributed as: - [x] PubSub communication with [`zenoh`](https://github.com/eclipse-zenoh/zenoh) diff --git a/docs/src/rust-client.md b/docs/src/library-vs-framework.md similarity index 100% rename from docs/src/rust-client.md rename to docs/src/library-vs-framework.md From 2de867fa69686f568ade85ee66952d7c61c6b272 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 10 Aug 2022 11:45:33 +0200 Subject: [PATCH 4/8] Adding api documentation --- README.md | 2 +- docs/src/c-api.md | 29 ++++++++ docs/src/communication-layer.md | 4 +- docs/src/dataflow-config.md | 50 +------------ docs/src/getting-started.md | 120 +++++++++++++++++++++++++++++--- docs/src/installation.md | 10 ++- docs/src/python-api.md | 42 +++++++++++ docs/src/rust-api.md | 73 +++++++++++++++++++ 8 files changed, 268 insertions(+), 62 deletions(-) create mode 100644 docs/src/c-api.md create mode 100644 docs/src/python-api.md create mode 100644 docs/src/rust-api.md diff --git a/README.md b/README.md index d1b80dc1..fc89949f 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Composability as: - [ ] language-agnostic: - [x] Rust - [x] C - - [ ] Python + - [x] Python - [ ] Isolated operator and node that can be reused. Low latency as: diff --git a/docs/src/c-api.md b/docs/src/c-api.md new file mode 100644 index 00000000..5a7f0f7c --- /dev/null +++ b/docs/src/c-api.md @@ -0,0 +1,29 @@ +# C API + +## Operator + +The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. + +### Try it out! + +- Create an `operator.c` file: +```c +{{#include ../../examples/c-operator/operator.c}} +``` + +- Copy `operator.h` header file: + +```bash +cp apis/c/operator/api.h . +``` + +- And compile your C operator: +```bash +clang -c operator.c +clang -shared -v operator.o -o operator.so -fPIC +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:47:52}} +``` \ No newline at end of file diff --git a/docs/src/communication-layer.md b/docs/src/communication-layer.md index bf06f231..0469270d 100644 --- a/docs/src/communication-layer.md +++ b/docs/src/communication-layer.md @@ -1,4 +1,4 @@ -# Middleware (communication) layer abstraction (MLA) +# [Middleware (communication) layer abstraction (MLA)](https://github.com/dora-rs/dora/discussions/53) `dora` needs to implement MLA as a separate crate to provides a middleware abstraction layer that enables scalable, high performance communications for inter async tasks, intra-process (OS threads), interprocess communication on a single computer node or between different nodes in a computer network. MLA needs to support different communication patterns: - publish-subscribe push / push pattern - the published message is pushed to subscribers @@ -29,4 +29,4 @@ MLA also needs to provide high level APIs: - send(key, msg, optional fields) - recv(key, optional fields)->value -More info here: https://github.com/dora-rs/dora/discussions/53 \ No newline at end of file +More info here: [#53](https://github.com/dora-rs/dora/discussions/53) \ No newline at end of file diff --git a/docs/src/dataflow-config.md b/docs/src/dataflow-config.md index cbf85535..9d881460 100644 --- a/docs/src/dataflow-config.md +++ b/docs/src/dataflow-config.md @@ -96,57 +96,11 @@ Each operator must specify exactly one implementation. The implementation must f ## Example -TODO: - - ```yaml -nodes: - - id: main - name: Main Node - description: Implements the main task - - operators: - # sources (only outputs) - - id: timer - name: Clock timer # optional, human-readable name - description: Send the time. - shared_library: path/to/timer.so - outputs: - - time - - id: camera-uuid-1 - name: Front camera - python: camera.py - outputs: - - image - - metadata - - # actions (inputs and outputs) - - id: timestamp - description: Add a watermark on the camera. - python: timestamp.py - inputs: - timestamp: timer/time - image: camera-uuid-1/image - outputs: - - image # with timestamp watermark - - # sinks (only inputs) - - id: logger - description: Sink the data into the logger. - python: logger.py - inputs: - image: timestamp/image - camera: camera-uuid-2/metadata - - - id: camera-uuid-2 - name: Back camera - run: camera_driver --kind back - outputs: - - image - - metadata - +{{#include ../../binaries/coordinator/examples/mini-dataflow.yml}} ``` + ## TODO: Integration with ROS 1/2 To integrate dora-rs operators with ROS1 or ROS2 operators, we plan to provide special _bridge operators_. These operators act as a sink in one dataflow framework and push all messages to a different dataflow framework, where they act as source. diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 7d561161..d0befd85 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -1,17 +1,117 @@ -### 2. Compile the Rust example operator: +### Create a Rust workspace + +- Initiate the workspace with: + +```bash +mkdir my_first_dataflow +cd my_first_dataflow +``` + +- Create the Cargo.toml file that will configure the entire workspace: + +`Cargo.toml` +```toml +[workspace] + +members = [ + "source_timer", +] +``` + +### Write your first node + +Let's write a node which sends the current time periodically. Let's make it after 100 iterations. The other nodes/operators will then exit as well because all sources closed. + +- Generate a new Rust binary (application): ```bash -cargo build --manifest-path ../examples/example-operator/Cargo.toml --release +cargo new source_timer ``` -### 3. Compile the C example operator: +with `Cargo.toml`: +```toml +[package] +name = "rust-node" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +dora-node-api = { git = "https://github.com/dora-rs/dora" } +time = "0.3.9" +``` + +with `src/main.rs`: +```rust +{{#include ../../binaries/coordinator/examples/nodes/rust/source_timer.rs}} +``` + +### Write your second node + +Let's write a `logger` which will print incoming data. + +- Generate a new Rust binary (application): + ```bash -cd ../../examples/c-operator -cp ../../apis/c/operator/api.h . -clang -c operator.c -clang -shared -v operator.o -o operator.so +cargo new sink_logger ``` -- Run the `mini-dataflow` example using `cargo run --release -- run examples/mini-dataflow.yml` - - This spawns a `timer` source, which sends the current time periodically, and a `logger` sink, which prints the incoming data. - - The `timer` will exit after 100 iterations. The other nodes/operators will then exit as well because all sources closed. +with `Cargo.toml`: +```toml +[package] +name = "sink_logger" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +dora-node-api = { git = "https://github.com/dora-rs/dora" } +time = "0.3.9" +``` + +with `src/main.rs`: +```rust +{{#include ../../binaries/coordinator/examples/nodes/rust/sink_logger.rs}} +``` + +- And modify the root `Cargo.toml`: +```toml= +[workspace] + +members = [ + "source_timer", + "sink_logger" +] +``` + + +### Write a graph definition + +Let's write the graph definition so that the nodes know who to communicate with. + +`mini-dataflow.yml` +```yaml +communication: + zenoh: + prefix: /foo + +nodes: + - id: timer + custom: + run: cargo run --release --bin source_timer + outputs: + - time + + - id: logger + custom: + run: cargo run --release --bin sink_logger + inputs: + time: timer/time +``` + +### Run it! + +- Run the `mini-dataflow`: +```bash +dora-coordinator run mini-dataflow.yml +``` diff --git a/docs/src/installation.md b/docs/src/installation.md index 8a28f502..88a21f3b 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -8,7 +8,9 @@ The `dora-coordinator` is responsible for reading the dataflow descriptor file a Build it using: ```bash -cargo build -p dora-coordinator --examples --release +git clone https://github.com/dora-rs/dora.git +cd dora +cargo build -p dora-coordinator --release ``` ### 2. Compile the dora-runtime for operators @@ -17,3 +19,9 @@ The `dora-runtime` is responsible for managing a set of operators. ```bash cargo build -p dora-runtime --release ``` + +### 3. Add those binaries to your path + +```bash +export PATH=$PATH:$(pwd)/target/release +``` \ No newline at end of file diff --git a/docs/src/python-api.md b/docs/src/python-api.md new file mode 100644 index 00000000..248ce03f --- /dev/null +++ b/docs/src/python-api.md @@ -0,0 +1,42 @@ +# Python API + +## Cutom Node + +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. + +### Try it out! + +- Install python node API: +```bash +cd apis/python/node +python3 -m venv .env +source .env/bin/activate +pip install maturin +maturin develop +``` + +- Create a python file called `printer.py`: +```python +{{#include ../../binaries/coordinator/examples/nodes/python/printer.py}} +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/graphs/python_test.yml:12:17}} +``` + +## Operator + +The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. + +### Try it out! + +- Create an operator python file called `op.py`: +```python +{{#include ../../examples/python-operator/op.py}} +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/graphs/mini-dataflow.yml:67:73}} +``` \ No newline at end of file diff --git a/docs/src/rust-api.md b/docs/src/rust-api.md new file mode 100644 index 00000000..5680ae60 --- /dev/null +++ b/docs/src/rust-api.md @@ -0,0 +1,73 @@ +# Rust API + +## Node + +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. + +### Try it out! + +- Generate a new Rust binary (application): + +```bash +cargo new source_timer +``` + +```toml +[package] +name = "source_timer" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +dora-node-api = { path = "../../apis/rust/node" } +time = "0.3.9" +``` + +`src/main.rs` +```rust +{{#include ../../binaries/coordinator/examples/nodes/rust/source_timer.rs}} +``` + +- Link it in your graph as: +```yaml + - id: timer + custom: + run: cargo run --release + outputs: + - time +``` + +## Operator + +The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. + +### Try it out! + +- Generate a new Rust library + +```bash +cargo new example-operator --lib +``` + +`Cargo.toml` +```toml +{{#include ../../examples/example-operator/Cargo.toml}} +``` + +`src/lib.rs` +```rust +{{#include ../../examples/example-operator/src/lib.rs}} +``` + +- Build it: +```bash +cargo build --release +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:38:46}} +``` + +This example can be found in `examples`. \ No newline at end of file From be7c4937526451d92bb95022875b931b9fe91a18 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Wed, 10 Aug 2022 11:46:20 +0200 Subject: [PATCH 5/8] fix typo in `SUMMARY.md` --- docs/src/SUMMARY.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 2d3fa62a..972a83f6 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -14,9 +14,12 @@ - [Overview](overview.md) A high level overview of dora's different components and the differences between operators and custom nodes. - [Dataflow Configuration](dataflow-config.md) Specification of dora's YAML-based dataflow configuration format. +- [Rust API](rust-api.md) +- [C API](c-api.md) +- [Python API](python-api.md) # Brainstorming Ideas - [State Management](state-management.md) Describes dora's state management capabilities for state sharing and recovery. -- [Librairy vs Framework](rust-client.md) +- [Library vs Framework](library-vs-framework.md) - [Middleware Layer Abstraction](./communication-layer.md) \ No newline at end of file From 41b40b5798926fef894e2e243a1b342e2abfdb7d Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Aug 2022 10:45:31 +0200 Subject: [PATCH 6/8] Adding method documentation and C node api --- docs/src/c-api.md | 89 ++++++++++++++++++++++++++++++++++++------ docs/src/python-api.md | 45 ++++++++++++++++++++- docs/src/rust-api.md | 44 +++++++++++++++++++++ 3 files changed, 166 insertions(+), 12 deletions(-) diff --git a/docs/src/c-api.md b/docs/src/c-api.md index 5a7f0f7c..740e655a 100644 --- a/docs/src/c-api.md +++ b/docs/src/c-api.md @@ -1,28 +1,95 @@ # C API -## Operator +## Custom Node -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. + +#### `init_dora_context_from_env` + +`init_dora_context_from_env` initiate a node from environment variables set by `dora-coordinator` + +```c +void *dora_context = init_dora_context_from_env(); +``` + +#### `dora_next_input` and `read_dora_input_data` + +`dora_next_input` and `read_dora_input_data` gives you the next input received. +```c +void *input = dora_next_input(dora_context); + +char *data; +size_t data_len; +read_dora_input_data(input, &data, &data_len); +``` + +#### `dora_send_output` + +`dora_send_output` send data from the node. + +```c +char out_id[] = "tick"; +dora_send_output(dora_context, out_id, strlen(out_id), &i, 1); +``` ### Try it out! -- Create an `operator.c` file: +- Create an `node.c` file: ```c -{{#include ../../examples/c-operator/operator.c}} +{{#include ../../examples/c-dataflow/node.c}} ``` -- Copy `operator.h` header file: +{{#include ../../examples/c-dataflow/README.md:26:35}} + + +## Operator + +The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. + +The operator definition is composed of 3 functions, `dora_init_operator` that initialise the operator and its context. `dora_drop_operator` that free the memory, and `dora_on_input` that action the logic of the operator on receiving an input. + +```c +int dora_init_operator(void **operator_context) +{ + void *context = malloc(1); + char *context_char = (char *)context; + *context_char = 0; + + *operator_context = context; + + return 0; +} + +void dora_drop_operator(void *operator_context) +{ + free(operator_context); +} -```bash -cp apis/c/operator/api.h . +int dora_on_input( + const char *id_start, + size_t id_len, + const char *data_start, + size_t data_len, + const int (*output_fn_raw)(const char *id_start, + size_t id_len, + const char *data_start, + size_t data_len, + const void *output_context), + void *output_context, + const void *operator_context) +{ + ... +} ``` +### Try it out! -- And compile your C operator: -```bash -clang -c operator.c -clang -shared -v operator.o -o operator.so -fPIC +- Create an `operator.c` file: +```c +{{#include ../../examples/c-dataflow/operator.c}} ``` +{{#include ../../examples/c-dataflow/README.md:40:46}} + - Link it in your graph as: ```yaml {{#include ../../binaries/coordinator/examples/mini-dataflow.yml:47:52}} diff --git a/docs/src/python-api.md b/docs/src/python-api.md index 248ce03f..d37c36a5 100644 --- a/docs/src/python-api.md +++ b/docs/src/python-api.md @@ -2,7 +2,37 @@ ## Cutom Node -The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. +#### `Node()` + +`Node()` initiate a node from environment variables set by `dora-coordinator` + +```python +from dora import Node() + +node = Node() +``` + +#### `.next()` or `__next__()` as an iterator + +`.next()` gives you the next input that the node has received. + +```python +input_id, value = node.next() + +# or + +for input_id, value in node: +``` + +#### `.send_output(output_id, data)` + +`send_output` send data from the node. + +```python +node.send_output("string", b"string") +``` + ### Try it out! @@ -29,6 +59,19 @@ maturin develop The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. +An operator requires an `on_input` method and requires to return a `DoraStatus` of 0 or 1, depending of it needs to continue or stop. + +```python +class Operator: + def on_input( + self, + input_id: str, + value: bytes, + send_output: Callable[[str, bytes], None], + ) -> DoraStatus: +``` + +> For Python, we recommend to allocate the operator on a single runtime. A runtime will share the same GIL with several operators making those operators run almost sequentially. See: [https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks](https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks) ### Try it out! - Create an operator python file called `op.py`: diff --git a/docs/src/rust-api.md b/docs/src/rust-api.md index 5680ae60..80b0165a 100644 --- a/docs/src/rust-api.md +++ b/docs/src/rust-api.md @@ -3,6 +3,29 @@ ## Node The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. +#### `DoraNode::init_from_env()` + +`DoraNode::init_from_env()` initiate a node from environment variables set by `dora-coordinator` + +```rust +let node = DoraNode::init_from_env().await?; +``` + +#### `.inputs()` + +`.inputs()` gives you a stream of input that you can access using `next()` on the input stream. + +```rust +let mut inputs = node.inputs().await?; +``` + +#### `.send_output(output_id, data)` + +`send_output` send data from the node. + +```rust +node.send_output(&data_id, data.as_bytes()).await?; +``` ### Try it out! @@ -42,6 +65,27 @@ time = "0.3.9" The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. +An operator requires to be registered and implement the `DoraOperator` trait, which is composed of an `on_input` method that defines the behaviour of the operator when there is an input. + +```rust +use dora_operator_api::{register_operator, DoraOperator, DoraOutputSender, DoraStatus}; + +register_operator!(ExampleOperator); + +#[derive(Debug, Default)] +struct ExampleOperator { + time: Option, +} + +impl DoraOperator for ExampleOperator { + fn on_input( + &mut self, + id: &str, + data: &[u8], + output_sender: &mut DoraOutputSender, + ) -> Result { +``` + ### Try it out! - Generate a new Rust library From 282983a864d9daa08a7270e9b6a750e86f8a587c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Aug 2022 11:46:05 +0200 Subject: [PATCH 7/8] Docs: making `Operator` recommended way of using `dora` This commit also add more information within the example section of the c api and take into account the PR review of #61. --- docs/src/c-api.md | 101 +++++++++++++++++++---------------- docs/src/installation.md | 2 + docs/src/python-api.md | 62 +++++++++++----------- docs/src/rust-api.md | 112 +++++++++++++++++++-------------------- 4 files changed, 145 insertions(+), 132 deletions(-) diff --git a/docs/src/c-api.md b/docs/src/c-api.md index 740e655a..0028f22e 100644 --- a/docs/src/c-api.md +++ b/docs/src/c-api.md @@ -1,57 +1,18 @@ # C API -## Custom Node - -The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. - -#### `init_dora_context_from_env` - -`init_dora_context_from_env` initiate a node from environment variables set by `dora-coordinator` - -```c -void *dora_context = init_dora_context_from_env(); -``` - -#### `dora_next_input` and `read_dora_input_data` - -`dora_next_input` and `read_dora_input_data` gives you the next input received. - -```c -void *input = dora_next_input(dora_context); - -char *data; -size_t data_len; -read_dora_input_data(input, &data, &data_len); -``` - -#### `dora_send_output` - -`dora_send_output` send data from the node. - -```c -char out_id[] = "tick"; -dora_send_output(dora_context, out_id, strlen(out_id), &i, 1); -``` -### Try it out! - -- Create an `node.c` file: -```c -{{#include ../../examples/c-dataflow/node.c}} -``` - -{{#include ../../examples/c-dataflow/README.md:26:35}} - - ## Operator -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. The operator definition is composed of 3 functions, `dora_init_operator` that initialise the operator and its context. `dora_drop_operator` that free the memory, and `dora_on_input` that action the logic of the operator on receiving an input. ```c int dora_init_operator(void **operator_context) { + // allocate a single byte to store a counter + // (the operator context pointer can be used to keep arbitrary data between calls) void *context = malloc(1); + char *context_char = (char *)context; *context_char = 0; @@ -78,7 +39,9 @@ int dora_on_input( void *output_context, const void *operator_context) { - ... + // handle the input ... + // (sending outputs is possible using `output_fn_raw`) + // (the `operator_context` is the pointer created in `dora_init_operator`, i.e., a counter in our case) } ``` ### Try it out! @@ -93,4 +56,52 @@ int dora_on_input( - Link it in your graph as: ```yaml {{#include ../../binaries/coordinator/examples/mini-dataflow.yml:47:52}} -``` \ No newline at end of file +``` + +## Custom Node + +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. + +#### `init_dora_context_from_env` + +`init_dora_context_from_env` initiate a node from environment variables set by `dora-coordinator` + +```c +void *dora_context = init_dora_context_from_env(); +``` + +#### `dora_next_input` + +`dora_next_input` waits for the next input. To extract the input ID and data, use `read_dora_input_id` and `read_dora_input_data` on the returned pointer. + +```c +void *input = dora_next_input(dora_context); + +// read out the ID as a UTF8-encoded string +char *id; +size_t id_len; +read_dora_input_id(input, &id, &id_len); + +// read out the data as a byte array +char *data; +size_t data_len; +read_dora_input_data(input, &data, &data_len); +``` + +#### `dora_send_output` + +`dora_send_output` send data from the node. + +```c +char out_id[] = "tick"; +char out_data[] = {0, 0, 0}; +dora_send_output(dora_context, out_id, strlen(out_id), &out_data, sizeof out_data); +``` +### Try it out! + +- Create an `node.c` file: +```c +{{#include ../../examples/c-dataflow/node.c}} +``` + +{{#include ../../examples/c-dataflow/README.md:26:35}} diff --git a/docs/src/installation.md b/docs/src/installation.md index 88a21f3b..9492b2c4 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -22,6 +22,8 @@ cargo build -p dora-runtime --release ### 3. Add those binaries to your path +This step is optional. You can also refer to the executables using their full path or copy them somewhere else. + ```bash export PATH=$PATH:$(pwd)/target/release ``` \ No newline at end of file diff --git a/docs/src/python-api.md b/docs/src/python-api.md index d37c36a5..0fd90291 100644 --- a/docs/src/python-api.md +++ b/docs/src/python-api.md @@ -1,6 +1,35 @@ # Python API -## Cutom Node +## Operator + +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. It is the recommended way of using `dora`. + +An operator requires an `on_input` method and requires to return a `DoraStatus` of 0 or 1, depending of it needs to continue or stop. + +```python +class Operator: + def on_input( + self, + input_id: str, + value: bytes, + send_output: Callable[[str, bytes], None], + ) -> DoraStatus: +``` + +> For Python, we recommend to allocate the operator on a single runtime. A runtime will share the same GIL with several operators making those operators run almost sequentially. See: [https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks](https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks) +### Try it out! + +- Create an operator python file called `op.py`: +```python +{{#include ../../examples/python-operator/op.py}} +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/graphs/mini-dataflow.yml:67:73}} +``` + +## Custom Node The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. #### `Node()` @@ -15,7 +44,7 @@ node = Node() #### `.next()` or `__next__()` as an iterator -`.next()` gives you the next input that the node has received. +`.next()` gives you the next input that the node has received. It will return `None` if there is no input available. ```python input_id, value = node.next() @@ -54,32 +83,3 @@ maturin develop ```yaml {{#include ../../binaries/coordinator/examples/graphs/python_test.yml:12:17}} ``` - -## Operator - -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. - -An operator requires an `on_input` method and requires to return a `DoraStatus` of 0 or 1, depending of it needs to continue or stop. - -```python -class Operator: - def on_input( - self, - input_id: str, - value: bytes, - send_output: Callable[[str, bytes], None], - ) -> DoraStatus: -``` - -> For Python, we recommend to allocate the operator on a single runtime. A runtime will share the same GIL with several operators making those operators run almost sequentially. See: [https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks](https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks) -### Try it out! - -- Create an operator python file called `op.py`: -```python -{{#include ../../examples/python-operator/op.py}} -``` - -- Link it in your graph as: -```yaml -{{#include ../../binaries/coordinator/examples/graphs/mini-dataflow.yml:67:73}} -``` \ No newline at end of file diff --git a/docs/src/rust-api.md b/docs/src/rust-api.md index 80b0165a..4d4dfac9 100644 --- a/docs/src/rust-api.md +++ b/docs/src/rust-api.md @@ -1,6 +1,61 @@ # Rust API -## Node +## Operator + +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. It is the recommended way of using `dora`. + +An operator requires to be registered and implement the `DoraOperator` trait. It is composed of an `on_input` method that defines the behaviour of the operator when there is an input. + +```rust +use dora_operator_api::{register_operator, DoraOperator, DoraOutputSender, DoraStatus}; + +register_operator!(ExampleOperator); + +#[derive(Debug, Default)] +struct ExampleOperator { + time: Option, +} + +impl DoraOperator for ExampleOperator { + fn on_input( + &mut self, + id: &str, + data: &[u8], + output_sender: &mut DoraOutputSender, + ) -> Result { +``` + +### Try it out! + +- Generate a new Rust library + +```bash +cargo new example-operator --lib +``` + +`Cargo.toml` +```toml +{{#include ../../examples/example-operator/Cargo.toml}} +``` + +`src/lib.rs` +```rust +{{#include ../../examples/example-operator/src/lib.rs}} +``` + +- Build it: +```bash +cargo build --release +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:38:46}} +``` + +This example can be found in `examples`. + +## Custom Node The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. #### `DoraNode::init_from_env()` @@ -60,58 +115,3 @@ time = "0.3.9" outputs: - time ``` - -## Operator - -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. - -An operator requires to be registered and implement the `DoraOperator` trait, which is composed of an `on_input` method that defines the behaviour of the operator when there is an input. - -```rust -use dora_operator_api::{register_operator, DoraOperator, DoraOutputSender, DoraStatus}; - -register_operator!(ExampleOperator); - -#[derive(Debug, Default)] -struct ExampleOperator { - time: Option, -} - -impl DoraOperator for ExampleOperator { - fn on_input( - &mut self, - id: &str, - data: &[u8], - output_sender: &mut DoraOutputSender, - ) -> Result { -``` - -### Try it out! - -- Generate a new Rust library - -```bash -cargo new example-operator --lib -``` - -`Cargo.toml` -```toml -{{#include ../../examples/example-operator/Cargo.toml}} -``` - -`src/lib.rs` -```rust -{{#include ../../examples/example-operator/src/lib.rs}} -``` - -- Build it: -```bash -cargo build --release -``` - -- Link it in your graph as: -```yaml -{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:38:46}} -``` - -This example can be found in `examples`. \ No newline at end of file From b892e9f5d2b2dc75e917c4035ca6a081c7ef3332 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Aug 2022 11:53:40 +0200 Subject: [PATCH 8/8] Fix `.next()` docs for python --- docs/src/python-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/python-api.md b/docs/src/python-api.md index 0fd90291..4cea2410 100644 --- a/docs/src/python-api.md +++ b/docs/src/python-api.md @@ -44,7 +44,7 @@ node = Node() #### `.next()` or `__next__()` as an iterator -`.next()` gives you the next input that the node has received. It will return `None` if there is no input available. +`.next()` gives you the next input that the node has received. It blocks until the next input becomes available. It will return `None` when all senders has been dropped. ```python input_id, value = node.next()