add tpcds-randomized-test to ci

This commit adds tpcds-randomized-test to ci. It relies on the duckdb cli for tpc-ds database generation. It also saves the artifacts if the test fails so we can reproduce issues.

Author: jayshrivastava

.github/actions/setup/action.yml

@@ -17,6 +17,13 @@ runs:
         echo $HOME
         echo ${{ runner.os }}
 
+    - shell: bash
+      run: |
+        # Create target directory if CARGO_TARGET_DIR is set
+        if [ -n "$CARGO_TARGET_DIR" ]; then
+          sudo mkdir -p "$CARGO_TARGET_DIR" && sudo chown -R $USER:$USER "$CARGO_TARGET_DIR"
+        fi
+
     - uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable

.github/workflows/ci.yml

@@ -9,6 +9,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  CARGO_TARGET_DIR: /mnt/target # use larger disk 
 
 concurrency:
   group: ${{ github.ref }}
@@ -32,13 +33,67 @@ jobs:
           lfs: true
       - uses: ./.github/actions/setup
       - run: cargo test --features integration
+      - name: Show disk usage
+        if: always()
+        run: |
+          echo "=== df -h ==="
+          df -h
+          echo "=== Top directories in workspace ==="
+          du -h . 2>/dev/null | sort -hr | head -n 20 || true
+          echo "=== cargo dirs ==="
+          du -sh ~/.cargo 2>/dev/null || true
+          du -sh ~/.cargo/registry 2>/dev/null || true
+          du -sh ~/.cargo/git 2>/dev/null || true
+          du -sh /mnt/target 2>/dev/null || true
+          du -sh target 2>/dev/null || true
+          echo "=== target/debug/deps sizes ==="
+          du -h /mnt/target/debug/deps 2>/dev/null | sort -hr | head -n 30 || true
+          du -h target/debug/deps 2>/dev/null | sort -hr | head -n 30 || true
+          echo "=== home directory ==="
+          du -sh ~ 2>/dev/null | head -n 10 || true
 
   tpch-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: cargo test --features tpch --test tpch_validation_test
+      - name: Clean up test data
+        run: |
+          rm -rf testdata/tpch/data/*
+          df -h
+
+  tpcds-randomized-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+      - uses: ./.github/actions/setup
+      - name: Install DuckDB CLI
+        run: |
+          curl https://install.duckdb.org | sh
+          mkdir -p $HOME/.local/bin
+          mv /home/runner/.duckdb/cli/latest/duckdb $HOME/.local/bin/
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - name: Run TPC-DS randomized test
+        id: test
+        run: cargo test --features tpcds --test tpc_ds_randomized
+        continue-on-error: true
+      - name: Upload test artifacts on failure
+        if: failure() || steps.test.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: tpcds-test-artifacts-${{ github.run_id }}
+          path: testdata/tpcds/data/**
+          retention-days: 7
+          if-no-files-found: ignore
+      - name: Clean up test data
+        run: |
+          rm -rf testdata/tpcds/data/*
+          rm -f $HOME/.local/bin/duckdb
+          rm -rf /home/runner/.duckdb
+          df -h
 
   format-check:
     runs-on: ubuntu-latest
@@ -48,3 +103,4 @@ jobs:
         with:
           components: rustfmt
       - run: cargo fmt --all -- --check
+