Add schema generator

Cargo.lock

@@ -2306,6 +2306,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde-with-expand-env",
+ "serde_json",
  "serde_yaml 0.9.30",
  "tokio",
  "tracing",
@@ -2523,8 +2524,6 @@ dependencies = [
  "dora-tracing",
  "eyre",
  "parquet",
- "schemars",
- "serde_json",
  "tokio",
 ]
 

libraries/core/Cargo.toml

@@ -21,3 +21,4 @@ serde-with-expand-env = "1.1.0"
 tokio = { version = "1.24.1", features = ["fs", "process", "sync"] }
 aligned-vec = { version = "0.5.0", features = ["serde"] }
 schemars = "0.8.19"
+serde_json = "1.0.117"

libraries/core/README.md

@@ -0,0 +1,7 @@
+# Core library for dora
+
+## Generating dora schema
+
+```bash
+cargo run -p dora-core generate_schemas
+```

libraries/core/src/bin/generate_schema.rs

@@ -0,0 +1,37 @@
+use std::{env, path::Path};
+
+use dora_core::descriptor::Descriptor;
+use schemars::schema_for;
+
+fn main() -> () {
+    let schema = schema_for!(Descriptor);
+    let raw_schema =
+        serde_json::to_string_pretty(&schema).expect("Could not serialize schema to json");
+    let raw_schema = raw_schema.replace(
+        "\"additionalProperties\": false",
+        "\"additionalProperties\": true",
+    );
+
+    // Remove `serde(from=` nested field as they are not handled properly by `schemars`
+    let raw_schema = raw_schema.replace(
+        "\"python\": {
+              \"$ref\": \"#/definitions/PythonSource\"
+            }",
+        "",
+    );
+    let raw_schema = raw_schema.replace(
+        "{
+            \"$ref\": \"#/definitions/Input\"
+          }",
+        "true",
+    );
+
+    // Get the Cargo root manifest directory
+    let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not set");
+
+    // Create the path for the new file next to Cargo.toml
+    let new_file_path = Path::new(&manifest_dir).join("dora-schema.json");
+
+    // write to file
+    std::fs::write(new_file_path, raw_schema).expect("Could not write schema to file");
+}