add tpcds-randomized-test to ci

This commit adds tpcds-randomized-test to ci. It relies on the duckdb cli for tpc-ds database generation. It also saves the artifacts if the test fails so we can reproduce issues.

Author: jayshrivastava

.github/workflows/ci.yml

@@ -39,6 +39,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: cargo test --features tpch --test tpch_validation_test
+      - name: Clean up test data
+        run: |
+          rm -rf testdata/tpch/data/*
+          df -h
 
   format-check:
     runs-on: ubuntu-latest

.github/workflows/nightly.yml

@@ -0,0 +1,52 @@
+name: Nightly Tests
+
+on:
+  schedule:
+    # Run at 9pm EST (2am UTC next day)
+    - cron: '0 2 * * *'
+  workflow_dispatch: # Allow manual triggering
+
+concurrency:
+  # Run tests sequentially to avoid using too much disk space
+  group: nightly
+  cancel-in-progress: false
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  SCALE_FACTOR: "1" # use tpcds scale factor 1
+
+jobs:
+  tpcds-randomized-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+      - uses: ./.github/actions/setup
+      - name: Install DuckDB CLI
+        run: |
+          curl https://install.duckdb.org | sh
+          mkdir -p $HOME/.local/bin
+          mv /home/runner/.duckdb/cli/latest/duckdb $HOME/.local/bin/
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - name: Run TPC-DS randomized test
+        id: test
+        run: cargo test --features tpcds --test tpc_ds_randomized
+        continue-on-error: true
+      - name: Upload test artifacts on failure
+        if: failure() || steps.test.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: tpcds-test-artifacts-${{ github.run_id }}
+          path: testdata/tpcds/data/**
+          retention-days: 7
+          if-no-files-found: ignore
+      - name: Free up disk space 
+        run: |
+          rm -rf testdata/tpcds/data/*
+          rm -f $HOME/.local/bin/duckdb
+          rm -rf /home/runner/.duckdb
+          cargo clean
+          df -h
+