Make CI/CD parallel for the node-hub

.github/workflows/node-hub-ci-cd.yml

@@ -11,9 +11,28 @@ on:
     types: [published]
 
 jobs:
-  ci:
+  find-jobs:
     runs-on: ubuntu-latest
+    name: Find Jobs
+    outputs:
+      folders: ${{ steps.jobs.outputs.folders }}
+    steps:
+      - uses: actions/checkout@v1
+
+      - id: jobs
+        uses: kmanimaran/list-folder-action@v4
+        with:
+          path: ./node-hub
 
+  ci:
+    runs-on: ubuntu-latest
+    needs: [find-jobs]
+    defaults:
+      run:
+        working-directory: node-hub/${{ matrix.folder }}
+    strategy:
+      matrix:
+        folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -53,11 +72,17 @@ jobs:
 
       - name: Run Linting and Tests
         run: |
-          chmod +x .github/workflows/node_hub_test.sh
-          .github/workflows/node_hub_test.sh
+          chmod +x ../../.github/workflows/node_hub_test.sh
+          ../../.github/workflows/node_hub_test.sh
 
   publish:
-    needs: [ci]
+    needs: [ci, find-jobs]
+    defaults:
+      run:
+        working-directory: ${{ matrix.folder }}
+    strategy:
+      matrix:
+        folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')
 
@@ -102,21 +127,20 @@ jobs:
           POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASS }}
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
-          for dir in node-hub/*/ ; do
-            if [ -d "$dir" ]; then
-              if [[ -f "$dir/Cargo.toml" && -f "$dir/pyproject.toml" ]]; then
-                echo "Publishing $dir using maturin..."
-                (cd "$dir" && poetry publish)
-              else
-                if [ -f "$dir/pyproject.toml" ]; then
-                  echo "Publishing $dir using Poetry..."
-                  (cd "$dir" && poetry publish --build)
-                fi
-              fi
-              
-              if [ -f "$dir/Cargo.toml" ]; then
-                echo "Publishing $dir using Cargo..."
-                (cd "$dir" && cargo publish)
-              fi
+          dir=$(pwd)
+          base_dir=$(basename "$dir")
+
+          if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
+            echo "Publishing $dir using maturin..."
+            poetry publish
+          else
+            if [ -f "pyproject.toml" ]; then
+              echo "Publishing $dir using Poetry..."
+              poetry publish --build
             fi
-          done
+          fi
+
+          if [ -f "Cargo.toml" ]; then
+            echo "Publishing $dir using Cargo..."
+            cargo publish
+          fi

.github/workflows/node_hub_test.sh

@@ -4,29 +4,27 @@ set -euo
 # List of ignored modules 
 ignored_folders=("dora-internvl" "dora-parler" "dora-keyboard" "dora-microphone" "terminal-input")
 
-for dir in node-hub/*/ ; do
-    # Get the base name of the directory (without the path)
-    base_dir=$(basename "$dir")
+# Get current working directory
+dir=$(pwd)
 
-    # Check if the directory name is in the ignored list
-    if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
-        echo "Skipping $base_dir as there is a hf model fetching issue..."
-        continue
-    fi
+# Get the base name of the directory (without the path)
+base_dir=$(basename "$dir")
 
-    if [ -d "$dir" ]; then
-        if [ -f "$dir/Cargo.toml" ]; then
-            echo "Running build and tests for Rust project in $dir..."
-            (cd "$dir" && cargo build)
-            (cd "$dir" && cargo test)
-        else
-            if [ -f "$dir/pyproject.toml" ]; then
-            echo "Running linting and tests for Python project in $dir..."
-            (cd "$dir" && pip install .)
-            (cd "$dir" && poetry run black --check .)
-            (cd "$dir" && poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py)
-            (cd "$dir" && poetry run pytest)
-            fi
-        fi   
-    fi
-done
+# Check if the directory name is in the ignored list
+if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
+    echo "Skipping $base_dir as we cannot test it on the CI..."
+else
+    if [ -f "$dir/Cargo.toml" ]; then
+        echo "Running build and tests for Rust project in $dir..."
+        cargo build
+        cargo test
+    else
+        if [ -f "$dir/pyproject.toml" ]; then
+        echo "Running linting and tests for Python project in $dir..."
+        pip install .
+        poetry run black --check .
+        poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py
+        poetry run pytest
+        fi
+    fi   
+fi