From 0ac5a722e36eda906aa55ecc3997f6f8677ffb76 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 28 Oct 2022 17:44:57 +0200 Subject: [PATCH 1/9] Implement a C++ operator API Based on Rust API using `cxx` for bridging. --- Cargo.lock | 26 ++--- Cargo.toml | 1 - .../c++/operator}/Cargo.toml | 4 +- apis/c++/operator/build.rs | 4 + .../c++/operator}/src/lib.rs | 17 ++-- apis/c/operator/operator_api.h | 2 +- examples/c++-dataflow/dataflow.yml | 2 +- .../c++-dataflow/operator-rust-api/build.rs | 10 -- .../operator-rust-api/operator.cc | 23 +++++ .../c++-dataflow/operator-rust-api/operator.h | 16 ++++ .../operator-rust-api/src/operator.cc | 22 ----- .../operator-rust-api/src/operator.h | 17 ---- examples/c++-dataflow/run.rs | 94 ++++++++++++++----- 13 files changed, 141 insertions(+), 97 deletions(-) rename {examples/c++-dataflow/operator-rust-api => apis/c++/operator}/Cargo.toml (81%) create mode 100644 apis/c++/operator/build.rs rename {examples/c++-dataflow/operator-rust-api => apis/c++/operator}/src/lib.rs (86%) delete mode 100644 examples/c++-dataflow/operator-rust-api/build.rs create mode 100644 examples/c++-dataflow/operator-rust-api/operator.cc create mode 100644 examples/c++-dataflow/operator-rust-api/operator.h delete mode 100644 examples/c++-dataflow/operator-rust-api/src/operator.cc delete mode 100644 examples/c++-dataflow/operator-rust-api/src/operator.h diff --git a/Cargo.lock b/Cargo.lock index 32e5a827..cf59baee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -746,19 +746,6 @@ dependencies = [ "syn", ] -[[package]] -name = "cxx-dataflow-example-operator-rust-api" -version = "0.1.0" -dependencies = [ - "cxx", - "cxx-build", - "dora-operator-api", - "eyre", - "futures", - "rand", - "tokio", -] - [[package]] name = "cxxbridge-flags" version = "1.0.73" @@ -1033,6 +1020,19 @@ dependencies = [ "dora-operator-api-types", ] +[[package]] +name = "dora-operator-api-cxx" +version = "0.1.0" +dependencies = [ + "cxx", + "cxx-build", + "dora-operator-api", + "eyre", + "futures", + "rand", + "tokio", +] + [[package]] name = "dora-operator-api-macros" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5075bf1d..ea5de4ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ members = [ "apis/rust/operator/types", "binaries/*", "examples/rust-dataflow/*", - "examples/c++-dataflow/operator-rust-api", "examples/iceoryx/*", "libraries/communication-layer", "libraries/core", diff --git a/examples/c++-dataflow/operator-rust-api/Cargo.toml b/apis/c++/operator/Cargo.toml similarity index 81% rename from examples/c++-dataflow/operator-rust-api/Cargo.toml rename to apis/c++/operator/Cargo.toml index cfc6bec4..980e07ca 100644 --- a/examples/c++-dataflow/operator-rust-api/Cargo.toml +++ b/apis/c++/operator/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "cxx-dataflow-example-operator-rust-api" +name = "dora-operator-api-cxx" version = "0.1.0" edition = "2021" [lib] -crate-type = ["cdylib"] +crate-type = ["staticlib"] [dependencies] cxx = "1.0.73" diff --git a/apis/c++/operator/build.rs b/apis/c++/operator/build.rs new file mode 100644 index 00000000..7b1aa53b --- /dev/null +++ b/apis/c++/operator/build.rs @@ -0,0 +1,4 @@ +fn main() { + let _ = cxx_build::bridge("src/lib.rs"); + println!("cargo:rerun-if-changed=src/lib.rs"); +} diff --git a/examples/c++-dataflow/operator-rust-api/src/lib.rs b/apis/c++/operator/src/lib.rs similarity index 86% rename from examples/c++-dataflow/operator-rust-api/src/lib.rs rename to apis/c++/operator/src/lib.rs index 8d3c9936..8aa49979 100644 --- a/examples/c++-dataflow/operator-rust-api/src/lib.rs +++ b/apis/c++/operator/src/lib.rs @@ -1,28 +1,29 @@ #![warn(unsafe_op_in_unsafe_fn)] use dora_operator_api::{self, register_operator, DoraOperator, DoraOutputSender, DoraStatus}; -use ffi::SendOutputResult; +use ffi::DoraSendOutputResult; #[cxx::bridge] #[allow(unsafe_op_in_unsafe_fn)] mod ffi { - struct OnInputResult { + struct DoraOnInputResult { error: String, stop: bool, } - struct SendOutputResult { + struct DoraSendOutputResult { error: String, } extern "Rust" { type OutputSender<'a, 'b>; - fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> SendOutputResult; + fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> DoraSendOutputResult; } unsafe extern "C++" { - include!("cxx-dataflow-example-operator-rust-api/src/operator.h"); + include!("operator.h"); + type Operator; fn new_operator() -> UniquePtr; @@ -32,19 +33,19 @@ mod ffi { id: &str, data: &[u8], output_sender: &mut OutputSender, - ) -> OnInputResult; + ) -> DoraOnInputResult; } } pub struct OutputSender<'a, 'b>(&'a mut DoraOutputSender<'b>); -fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> SendOutputResult { +fn send_output(sender: &mut OutputSender, id: &str, data: &[u8]) -> DoraSendOutputResult { let error = sender .0 .send(id.into(), data.to_owned()) .err() .unwrap_or_default(); - SendOutputResult { error } + DoraSendOutputResult { error } } register_operator!(OperatorWrapper); diff --git a/apis/c/operator/operator_api.h b/apis/c/operator/operator_api.h index cc67c567..cc105516 100644 --- a/apis/c/operator/operator_api.h +++ b/apis/c/operator/operator_api.h @@ -23,7 +23,7 @@ extern "C" const SendOutput_t *send_output, void *operator_context); - void __dora_type_assertions() + static void __dora_type_assertions() { DoraInitOperator_t __dora_init_operator = {.init_operator = dora_init_operator}; DoraDropOperator_t __dora_drop_operator = {.drop_operator = dora_drop_operator}; diff --git a/examples/c++-dataflow/dataflow.yml b/examples/c++-dataflow/dataflow.yml index ffa0f061..960866e6 100644 --- a/examples/c++-dataflow/dataflow.yml +++ b/examples/c++-dataflow/dataflow.yml @@ -21,7 +21,7 @@ nodes: - id: runtime-node operators: - id: operator-rust-api - shared-library: ../../target/debug/cxx_dataflow_example_operator_rust_api + shared-library: build/operator_rust_api inputs: counter_1: cxx-node-c-api/counter counter_2: cxx-node-rust-api/counter diff --git a/examples/c++-dataflow/operator-rust-api/build.rs b/examples/c++-dataflow/operator-rust-api/build.rs deleted file mode 100644 index c8b861a4..00000000 --- a/examples/c++-dataflow/operator-rust-api/build.rs +++ /dev/null @@ -1,10 +0,0 @@ -fn main() { - cxx_build::bridge("src/lib.rs") // returns a cc::Build - .file("src/operator.cc") - .flag_if_supported("-std=c++14") - .compile("cxx-example-dataflow-operator"); - - println!("cargo:rerun-if-changed=src/lib.rs"); - println!("cargo:rerun-if-changed=src/operator.cc"); - println!("cargo:rerun-if-changed=src/operator.h"); -} diff --git a/examples/c++-dataflow/operator-rust-api/operator.cc b/examples/c++-dataflow/operator-rust-api/operator.cc new file mode 100644 index 00000000..2b812188 --- /dev/null +++ b/examples/c++-dataflow/operator-rust-api/operator.cc @@ -0,0 +1,23 @@ +#include "operator.h" +#include +#include +#include "../build/dora-operator-api.h" + +Operator::Operator() {} + +std::unique_ptr new_operator() +{ + return std::make_unique(); +} + +DoraOnInputResult on_input(Operator &op, rust::Str id, rust::Slice data, OutputSender &output_sender) +{ + op.counter += 1; + std::cout << "Rust API operator received input `" << id.data() << "` with data `" << (unsigned int)data[0] << "` (internal counter: " << (unsigned int)op.counter << ")" << std::endl; + + std::vector out_vec{op.counter}; + rust::Slice out_slice{out_vec.data(), out_vec.size()}; + auto send_result = send_output(output_sender, rust::Str("status"), out_slice); + DoraOnInputResult result = {send_result.error, false}; + return result; +} diff --git a/examples/c++-dataflow/operator-rust-api/operator.h b/examples/c++-dataflow/operator-rust-api/operator.h new file mode 100644 index 00000000..9b5e3ab2 --- /dev/null +++ b/examples/c++-dataflow/operator-rust-api/operator.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include "../../../apis/c/operator/operator_api.h" + +class Operator +{ +public: + Operator(); + unsigned char counter; +}; + +#include "../build/dora-operator-api.h" + +std::unique_ptr new_operator(); + +DoraOnInputResult on_input(Operator &op, rust::Str id, rust::Slice data, OutputSender &output_sender); diff --git a/examples/c++-dataflow/operator-rust-api/src/operator.cc b/examples/c++-dataflow/operator-rust-api/src/operator.cc deleted file mode 100644 index d1eb244c..00000000 --- a/examples/c++-dataflow/operator-rust-api/src/operator.cc +++ /dev/null @@ -1,22 +0,0 @@ -#include "cxx-dataflow-example-operator-rust-api/src/operator.h" -#include "cxx-dataflow-example-operator-rust-api/src/lib.rs.h" -#include - -Operator::Operator() {} - -std::unique_ptr new_operator() -{ - return std::make_unique(); -} - -OnInputResult on_input(Operator &op, rust::Str id, rust::Slice data, OutputSender &output_sender) -{ - op.counter += 1; - std::cout << "Rust API operator received input `" << id << "` with data `" << (unsigned int)data[0] << "` (internal counter: " << (unsigned int)op.counter << ")" << std::endl; - - std::vector out_vec{op.counter}; - rust::Slice out_slice{out_vec.data(), out_vec.size()}; - auto send_result = send_output(output_sender, rust::Str("status"), out_slice); - OnInputResult result = {send_result.error, false}; - return result; -} diff --git a/examples/c++-dataflow/operator-rust-api/src/operator.h b/examples/c++-dataflow/operator-rust-api/src/operator.h deleted file mode 100644 index bdaecbf4..00000000 --- a/examples/c++-dataflow/operator-rust-api/src/operator.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once -#include "rust/cxx.h" -#include - -class Operator -{ -public: - Operator(); - unsigned char counter; -}; - -std::unique_ptr new_operator(); - -struct OnInputResult; -struct OutputSender; - -OnInputResult on_input(Operator &op, rust::Str, rust::Slice, OutputSender &output_sender); diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 594beb8c..d0f9e58c 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -15,19 +15,42 @@ async fn main() -> eyre::Result<()> { tokio::fs::create_dir_all("build").await?; let build_dir = Path::new("build"); - build_package("cxx-dataflow-example-operator-rust-api").await?; + build_package("dora-operator-api-cxx").await?; + let operator_cxxbridge = target + .join("cxxbridge") + .join("dora-operator-api-cxx") + .join("src"); + tokio::fs::copy( + operator_cxxbridge.join("lib.rs.cc"), + build_dir.join("operator-bridge.cc"), + ) + .await?; + tokio::fs::copy( + operator_cxxbridge.join("lib.rs.h"), + build_dir.join("dora-operator-api.h"), + ) + .await?; build_package("dora-node-api-cxx").await?; - let cxxbridge = target + let node_cxxbridge = target .join("cxxbridge") .join("dora-node-api-cxx") .join("src"); - tokio::fs::copy(cxxbridge.join("lib.rs.cc"), build_dir.join("bridge.cc")).await?; tokio::fs::copy( - cxxbridge.join("lib.rs.h"), + node_cxxbridge.join("lib.rs.cc"), + build_dir.join("node-bridge.cc"), + ) + .await?; + tokio::fs::copy( + node_cxxbridge.join("lib.rs.h"), build_dir.join("dora-node-api.h"), ) .await?; + tokio::fs::write( + build_dir.join("operator.h"), + r###"#include "../operator-rust-api/operator.h""###, + ) + .await?; build_package("dora-node-api-c").await?; build_package("dora-operator-api-c").await?; @@ -35,7 +58,7 @@ async fn main() -> eyre::Result<()> { root, &[ &dunce::canonicalize(Path::new("node-rust-api").join("main.cc"))?, - &dunce::canonicalize(build_dir.join("bridge.cc"))?, + &dunce::canonicalize(build_dir.join("node-bridge.cc"))?, ], "node_rust_api", &["-l", "dora_node_api_cxx"], @@ -51,8 +74,26 @@ async fn main() -> eyre::Result<()> { ) .await?; build_cxx_operator( - &dunce::canonicalize(Path::new("operator-c-api").join("operator.cc"))?, + &[ + &dunce::canonicalize(Path::new("operator-rust-api").join("operator.cc"))?, + &dunce::canonicalize(build_dir.join("operator-bridge.cc"))?, + ], + "operator_rust_api", + &[ + "-Wl,--export-dynamic-symbol,dora_on_input", + "-l", + "dora_operator_api_cxx", + "-L", + &root.join("target").join("debug").to_str().unwrap(), + ], + ) + .await?; + build_cxx_operator( + &[&dunce::canonicalize( + Path::new("operator-c-api").join("operator.cc"), + )?], "operator_c_api", + &[], ) .await?; @@ -151,27 +192,36 @@ async fn build_cxx_node( Ok(()) } -async fn build_cxx_operator(path: &Path, out_name: &str) -> eyre::Result<()> { - let object_file_path = Path::new("../build").join(out_name).with_extension("o"); - - let mut compile = tokio::process::Command::new("clang++"); - compile.arg("-c").arg(path); - compile.arg("-std=c++17"); - compile.arg("-o").arg(&object_file_path); - #[cfg(unix)] - compile.arg("-fPIC"); - if let Some(parent) = path.parent() { - compile.current_dir(parent); +async fn build_cxx_operator( + paths: &[&Path], + out_name: &str, + link_args: &[&str], +) -> eyre::Result<()> { + let mut object_file_paths = Vec::new(); + + for path in paths { + let mut compile = tokio::process::Command::new("clang++"); + compile.arg("-c").arg(path); + compile.arg("-std=c++17"); + let object_file_path = path.with_extension("o"); + compile.arg("-o").arg(&object_file_path); + #[cfg(unix)] + compile.arg("-fPIC"); + if let Some(parent) = path.parent() { + compile.current_dir(parent); + } + if !compile.status().await?.success() { + bail!("failed to compile cxx operator"); + }; + object_file_paths.push(object_file_path); } - if !compile.status().await?.success() { - bail!("failed to compile cxx operator"); - }; let mut link = tokio::process::Command::new("clang++"); - link.arg("-shared").arg(&object_file_path); + link.arg("-shared").args(&object_file_paths); + link.args(link_args); link.arg("-o") .arg(Path::new("../build").join(library_filename(out_name))); - if let Some(parent) = path.parent() { + if let Some(parent) = paths[0].parent() { link.current_dir(parent); } if !link.status().await?.success() { From fa1c1388f6aa1a582ccc26bed5264c6e247dfd1a Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 28 Oct 2022 17:47:15 +0200 Subject: [PATCH 2/9] Ignore generated object files --- examples/c++-dataflow/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/c++-dataflow/.gitignore diff --git a/examples/c++-dataflow/.gitignore b/examples/c++-dataflow/.gitignore new file mode 100644 index 00000000..5761abcf --- /dev/null +++ b/examples/c++-dataflow/.gitignore @@ -0,0 +1 @@ +*.o From 34b57e964c32eab0213023200c5e0887d6b64642 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 28 Oct 2022 17:47:46 +0200 Subject: [PATCH 3/9] Set default visibility for `EXPORT` symbols on UNIX --- apis/c/operator/operator_api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/c/operator/operator_api.h b/apis/c/operator/operator_api.h index cc105516..4d7414cf 100644 --- a/apis/c/operator/operator_api.h +++ b/apis/c/operator/operator_api.h @@ -11,7 +11,7 @@ extern "C" #ifdef _WIN32 #define EXPORT __declspec(dllexport) #else -#define EXPORT +#define EXPORT __attribute__((visibility("default"))) #endif EXPORT DoraInitResult_t dora_init_operator(void); From f0f13c9bcf09dd24b82b2acbc3ccf7a7442d2266 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 2 Nov 2022 17:05:22 +0100 Subject: [PATCH 4/9] Remove `export-dynamic-symbol` linker argument --- examples/c++-dataflow/run.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index d0f9e58c..db74f5c8 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -80,7 +80,6 @@ async fn main() -> eyre::Result<()> { ], "operator_rust_api", &[ - "-Wl,--export-dynamic-symbol,dora_on_input", "-l", "dora_operator_api_cxx", "-L", From 3ee91b98181146f59449856f8f40df4d2e5b00f3 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 2 Nov 2022 17:33:38 +0100 Subject: [PATCH 5/9] Link dora staticlib after system libraries --- examples/c++-dataflow/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index db74f5c8..80012e26 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -127,7 +127,6 @@ async fn build_cxx_node( let mut clang = tokio::process::Command::new("clang++"); clang.args(paths); clang.arg("-std=c++17"); - clang.args(args); #[cfg(target_os = "linux")] { clang.arg("-l").arg("m"); @@ -177,6 +176,7 @@ async fn build_cxx_node( clang.arg("-l").arg("c"); clang.arg("-l").arg("m"); } + clang.args(args); clang.arg("-L").arg(root.join("target").join("debug")); clang .arg("--output") From 6aaf5960b1112da33ff61afd05499670f1ef2d05 Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 4 Nov 2022 10:22:42 +0100 Subject: [PATCH 6/9] Link c++ operators with various system libraries on Windows --- examples/c++-dataflow/run.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 80012e26..6a7cdc82 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -218,6 +218,38 @@ async fn build_cxx_operator( let mut link = tokio::process::Command::new("clang++"); link.arg("-shared").args(&object_file_paths); link.args(link_args); + #[cfg(target_os = "windows")] + { + link.arg("-ladvapi32"); + link.arg("-luserenv"); + link.arg("-lkernel32"); + link.arg("-lws2_32"); + link.arg("-lbcrypt"); + link.arg("-lncrypt"); + link.arg("-lschannel"); + link.arg("-lntdll"); + link.arg("-liphlpapi"); + + link.arg("-lcfgmgr32"); + link.arg("-lcredui"); + link.arg("-lcrypt32"); + link.arg("-lcryptnet"); + link.arg("-lfwpuclnt"); + link.arg("-lgdi32"); + link.arg("-lmsimg32"); + link.arg("-lmswsock"); + link.arg("-lole32"); + link.arg("-lopengl32"); + link.arg("-lsecur32"); + link.arg("-lshell32"); + link.arg("-lsynchronization"); + link.arg("-luser32"); + link.arg("-lwinspool"); + + link.arg("-Wl,-nodefaultlib:libcmt"); + link.arg("-D_DLL"); + link.arg("-lmsvcrt"); + } link.arg("-o") .arg(Path::new("../build").join(library_filename(out_name))); if let Some(parent) = paths[0].parent() { From 857c83a0d910a5e63f31890ad3bfd8ffd1b7158e Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 4 Nov 2022 11:28:12 +0100 Subject: [PATCH 7/9] Link with `-fms-runtime-lib=static` on Windows --- examples/c++-dataflow/run.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 6a7cdc82..806cec91 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -249,6 +249,7 @@ async fn build_cxx_operator( link.arg("-Wl,-nodefaultlib:libcmt"); link.arg("-D_DLL"); link.arg("-lmsvcrt"); + link.arg("-fms-runtime-lib=static"); } link.arg("-o") .arg(Path::new("../build").join(library_filename(out_name))); From 3eb1144952d6d31970809b2dcdae92c0970a59eb Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Fri, 4 Nov 2022 11:44:58 +0100 Subject: [PATCH 8/9] Don't consider the c++-operator-api library for unit tests --- apis/c++/operator/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/apis/c++/operator/src/lib.rs b/apis/c++/operator/src/lib.rs index 8aa49979..ebf20c49 100644 --- a/apis/c++/operator/src/lib.rs +++ b/apis/c++/operator/src/lib.rs @@ -1,3 +1,4 @@ +#![cfg(not(test))] #![warn(unsafe_op_in_unsafe_fn)] use dora_operator_api::{self, register_operator, DoraOperator, DoraOutputSender, DoraStatus}; From ff179c3b933d48c66f7bbcc4676f4fcd566f927f Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Thu, 10 Nov 2022 12:28:23 +0100 Subject: [PATCH 9/9] Early-exit the c++ example on Windows --- examples/c++-dataflow/run.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/c++-dataflow/run.rs b/examples/c++-dataflow/run.rs index 806cec91..b76c52d3 100644 --- a/examples/c++-dataflow/run.rs +++ b/examples/c++-dataflow/run.rs @@ -7,6 +7,11 @@ use std::{ #[tokio::main] async fn main() -> eyre::Result<()> { + if cfg!(windows) { + eprintln!("The c++ example does not work on Windows currently because of a linker error"); + return Ok(()); + } + let root = Path::new(env!("CARGO_MANIFEST_DIR")); let target = root.join("target"); std::env::set_current_dir(root.join(file!()).parent().unwrap())