From 282983a864d9daa08a7270e9b6a750e86f8a587c Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 11 Aug 2022 11:46:05 +0200 Subject: [PATCH] Docs: making `Operator` recommended way of using `dora` This commit also add more information within the example section of the c api and take into account the PR review of #61. --- docs/src/c-api.md | 101 +++++++++++++++++++---------------- docs/src/installation.md | 2 + docs/src/python-api.md | 62 +++++++++++----------- docs/src/rust-api.md | 112 +++++++++++++++++++-------------------- 4 files changed, 145 insertions(+), 132 deletions(-) diff --git a/docs/src/c-api.md b/docs/src/c-api.md index 740e655a..0028f22e 100644 --- a/docs/src/c-api.md +++ b/docs/src/c-api.md @@ -1,57 +1,18 @@ # C API -## Custom Node - -The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. - -#### `init_dora_context_from_env` - -`init_dora_context_from_env` initiate a node from environment variables set by `dora-coordinator` - -```c -void *dora_context = init_dora_context_from_env(); -``` - -#### `dora_next_input` and `read_dora_input_data` - -`dora_next_input` and `read_dora_input_data` gives you the next input received. - -```c -void *input = dora_next_input(dora_context); - -char *data; -size_t data_len; -read_dora_input_data(input, &data, &data_len); -``` - -#### `dora_send_output` - -`dora_send_output` send data from the node. - -```c -char out_id[] = "tick"; -dora_send_output(dora_context, out_id, strlen(out_id), &i, 1); -``` -### Try it out! - -- Create an `node.c` file: -```c -{{#include ../../examples/c-dataflow/node.c}} -``` - -{{#include ../../examples/c-dataflow/README.md:26:35}} - - ## Operator -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. The operator definition is composed of 3 functions, `dora_init_operator` that initialise the operator and its context. `dora_drop_operator` that free the memory, and `dora_on_input` that action the logic of the operator on receiving an input. ```c int dora_init_operator(void **operator_context) { + // allocate a single byte to store a counter + // (the operator context pointer can be used to keep arbitrary data between calls) void *context = malloc(1); + char *context_char = (char *)context; *context_char = 0; @@ -78,7 +39,9 @@ int dora_on_input( void *output_context, const void *operator_context) { - ... + // handle the input ... + // (sending outputs is possible using `output_fn_raw`) + // (the `operator_context` is the pointer created in `dora_init_operator`, i.e., a counter in our case) } ``` ### Try it out! @@ -93,4 +56,52 @@ int dora_on_input( - Link it in your graph as: ```yaml {{#include ../../binaries/coordinator/examples/mini-dataflow.yml:47:52}} -``` \ No newline at end of file +``` + +## Custom Node + +The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. + +#### `init_dora_context_from_env` + +`init_dora_context_from_env` initiate a node from environment variables set by `dora-coordinator` + +```c +void *dora_context = init_dora_context_from_env(); +``` + +#### `dora_next_input` + +`dora_next_input` waits for the next input. To extract the input ID and data, use `read_dora_input_id` and `read_dora_input_data` on the returned pointer. + +```c +void *input = dora_next_input(dora_context); + +// read out the ID as a UTF8-encoded string +char *id; +size_t id_len; +read_dora_input_id(input, &id, &id_len); + +// read out the data as a byte array +char *data; +size_t data_len; +read_dora_input_data(input, &data, &data_len); +``` + +#### `dora_send_output` + +`dora_send_output` send data from the node. + +```c +char out_id[] = "tick"; +char out_data[] = {0, 0, 0}; +dora_send_output(dora_context, out_id, strlen(out_id), &out_data, sizeof out_data); +``` +### Try it out! + +- Create an `node.c` file: +```c +{{#include ../../examples/c-dataflow/node.c}} +``` + +{{#include ../../examples/c-dataflow/README.md:26:35}} diff --git a/docs/src/installation.md b/docs/src/installation.md index 88a21f3b..9492b2c4 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -22,6 +22,8 @@ cargo build -p dora-runtime --release ### 3. Add those binaries to your path +This step is optional. You can also refer to the executables using their full path or copy them somewhere else. + ```bash export PATH=$PATH:$(pwd)/target/release ``` \ No newline at end of file diff --git a/docs/src/python-api.md b/docs/src/python-api.md index d37c36a5..0fd90291 100644 --- a/docs/src/python-api.md +++ b/docs/src/python-api.md @@ -1,6 +1,35 @@ # Python API -## Cutom Node +## Operator + +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. It is the recommended way of using `dora`. + +An operator requires an `on_input` method and requires to return a `DoraStatus` of 0 or 1, depending of it needs to continue or stop. + +```python +class Operator: + def on_input( + self, + input_id: str, + value: bytes, + send_output: Callable[[str, bytes], None], + ) -> DoraStatus: +``` + +> For Python, we recommend to allocate the operator on a single runtime. A runtime will share the same GIL with several operators making those operators run almost sequentially. See: [https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks](https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks) +### Try it out! + +- Create an operator python file called `op.py`: +```python +{{#include ../../examples/python-operator/op.py}} +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/graphs/mini-dataflow.yml:67:73}} +``` + +## Custom Node The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. #### `Node()` @@ -15,7 +44,7 @@ node = Node() #### `.next()` or `__next__()` as an iterator -`.next()` gives you the next input that the node has received. +`.next()` gives you the next input that the node has received. It will return `None` if there is no input available. ```python input_id, value = node.next() @@ -54,32 +83,3 @@ maturin develop ```yaml {{#include ../../binaries/coordinator/examples/graphs/python_test.yml:12:17}} ``` - -## Operator - -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. - -An operator requires an `on_input` method and requires to return a `DoraStatus` of 0 or 1, depending of it needs to continue or stop. - -```python -class Operator: - def on_input( - self, - input_id: str, - value: bytes, - send_output: Callable[[str, bytes], None], - ) -> DoraStatus: -``` - -> For Python, we recommend to allocate the operator on a single runtime. A runtime will share the same GIL with several operators making those operators run almost sequentially. See: [https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks](https://docs.rs/pyo3/latest/pyo3/marker/struct.Python.html#deadlocks) -### Try it out! - -- Create an operator python file called `op.py`: -```python -{{#include ../../examples/python-operator/op.py}} -``` - -- Link it in your graph as: -```yaml -{{#include ../../binaries/coordinator/examples/graphs/mini-dataflow.yml:67:73}} -``` \ No newline at end of file diff --git a/docs/src/rust-api.md b/docs/src/rust-api.md index 80b0165a..4d4dfac9 100644 --- a/docs/src/rust-api.md +++ b/docs/src/rust-api.md @@ -1,6 +1,61 @@ # Rust API -## Node +## Operator + +The operator API is a framework for you to implement. The implemented operator will be managed by `dora`. This framework enable us to make optimisation and provide advanced features. It is the recommended way of using `dora`. + +An operator requires to be registered and implement the `DoraOperator` trait. It is composed of an `on_input` method that defines the behaviour of the operator when there is an input. + +```rust +use dora_operator_api::{register_operator, DoraOperator, DoraOutputSender, DoraStatus}; + +register_operator!(ExampleOperator); + +#[derive(Debug, Default)] +struct ExampleOperator { + time: Option, +} + +impl DoraOperator for ExampleOperator { + fn on_input( + &mut self, + id: &str, + data: &[u8], + output_sender: &mut DoraOutputSender, + ) -> Result { +``` + +### Try it out! + +- Generate a new Rust library + +```bash +cargo new example-operator --lib +``` + +`Cargo.toml` +```toml +{{#include ../../examples/example-operator/Cargo.toml}} +``` + +`src/lib.rs` +```rust +{{#include ../../examples/example-operator/src/lib.rs}} +``` + +- Build it: +```bash +cargo build --release +``` + +- Link it in your graph as: +```yaml +{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:38:46}} +``` + +This example can be found in `examples`. + +## Custom Node The custom node API allow you to integrate `dora` into your application. It allows you to retrieve input and send output in any fashion you want. #### `DoraNode::init_from_env()` @@ -60,58 +115,3 @@ time = "0.3.9" outputs: - time ``` - -## Operator - -The operator API gives you a framework for operator that is going to be managed by `dora`. This framework enable us to make optimisation and provide advanced features. - -An operator requires to be registered and implement the `DoraOperator` trait, which is composed of an `on_input` method that defines the behaviour of the operator when there is an input. - -```rust -use dora_operator_api::{register_operator, DoraOperator, DoraOutputSender, DoraStatus}; - -register_operator!(ExampleOperator); - -#[derive(Debug, Default)] -struct ExampleOperator { - time: Option, -} - -impl DoraOperator for ExampleOperator { - fn on_input( - &mut self, - id: &str, - data: &[u8], - output_sender: &mut DoraOutputSender, - ) -> Result { -``` - -### Try it out! - -- Generate a new Rust library - -```bash -cargo new example-operator --lib -``` - -`Cargo.toml` -```toml -{{#include ../../examples/example-operator/Cargo.toml}} -``` - -`src/lib.rs` -```rust -{{#include ../../examples/example-operator/src/lib.rs}} -``` - -- Build it: -```bash -cargo build --release -``` - -- Link it in your graph as: -```yaml -{{#include ../../binaries/coordinator/examples/mini-dataflow.yml:38:46}} -``` - -This example can be found in `examples`. \ No newline at end of file