From 56756758f2fbad26027b211d94bb75be504c9fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Finbarrs=20=E1=BB=8Ck=E1=BA=B9=CC=81t=C3=BAnj=C3=AD?= Date: Mon, 21 Oct 2024 11:14:18 +0100 Subject: [PATCH 1/7] python_sql_workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Finbarrs Ọkẹ́túnjí --- .github/workflows/python-lint.yaml | 32 ++++++ .github/workflows/sql-validation.yaml | 156 ++++++++++++++++++++++++++ LICENSE | 2 +- 3 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-lint.yaml create mode 100644 .github/workflows/sql-validation.yaml diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml new file mode 100644 index 0000000..e42b01f --- /dev/null +++ b/.github/workflows/python-lint.yaml @@ -0,0 +1,32 @@ +name: Linting + +on: + push: + branches: [develop] + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff and save results + run: | + timestamp=$(date +"%Y%m%d_%H%M%S") + ruff check . > logs/lint_result_${timestamp}.txt + - name: Run Ruff Format + run: ruff format --check . + - name: Upload lint results + uses: actions/upload-artifact@v3 + with: + name: lint-results + path: logs/lint_result_*.txt diff --git a/.github/workflows/sql-validation.yaml b/.github/workflows/sql-validation.yaml new file mode 100644 index 0000000..2ab9d6b --- /dev/null +++ b/.github/workflows/sql-validation.yaml @@ -0,0 +1,156 @@ +name: Validation, Reporting, and Cleanup + +on: + push: + branches: [develop] + pull_request: + branches: [main] + schedule: + - cron: "0 2 * * 5" # Run every Friday at 2 AM UTC + +jobs: + validate-and-report: + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install sqlfluff==3.2.0 sqlparse==0.5.1 + + - name: Create reports directory + run: mkdir -p reports + + - name: Lint Oracle files + run: | + echo "Linting Oracle files:" + if [ -d "oracle" ]; then + echo "Oracle directory found. Contents:" + ls -R oracle + echo "Running SQLFluff on Oracle files:" + sqlfluff lint --dialect oracle --format yaml oracle || true + echo "Saving lint results:" + if sqlfluff lint --dialect oracle --format yaml oracle > reports/oracle_lint_report.yaml; then + echo "Oracle linting completed successfully" + else + echo "Oracle linting encountered issues, check the report for details" + fi + echo "Oracle lint report:" + cat reports/oracle_lint_report.yaml + else + echo "No Oracle directory found" + fi + + - name: Lint MySQL files + run: | + echo "Linting MySQL files:" + if [ -d "mysql" ]; then + echo "MySQL directory found. Contents:" + ls -R mysql + echo "Running SQLFluff on MySQL files:" + sqlfluff lint --dialect mysql --format yaml mysql || true + echo "Saving lint results:" + if sqlfluff lint --dialect mysql --format yaml mysql > reports/mysql_lint_report.yaml; then + echo "MySQL linting completed successfully" + else + echo "MySQL linting encountered issues, check the report for details" + fi + echo "MySQL lint report:" + cat reports/mysql_lint_report.yaml + else + echo "No MySQL directory found" + fi + + - name: Check SQL syntax + run: | + echo "Checking SQL syntax:" + touch reports/syntax_report.txt + find_output=$(find . -name "*.sql") + if [ -z "$find_output" ]; then + echo "No SQL files found" | tee -a reports/syntax_report.txt + else + while IFS= read -r file; do + echo "Checking $file" | tee -a reports/syntax_report.txt + if ! cat "$file" | sqlparse --validate >> reports/syntax_report.txt 2>&1; then + echo "Error validating $file" | tee -a reports/syntax_report.txt + fi + echo "" >> reports/syntax_report.txt + done <<< "$find_output" + fi + echo "Syntax check complete. Report contents:" + cat reports/syntax_report.txt + + - name: Generate Markdown Report + run: | + report_file="reports/report-$(date +'%Y%m%d-%H%M%S').md" + echo "# SQL Validation Report" > $report_file + echo "## MySQL Linting Results" >> $report_file + echo '```yaml' >> $report_file + cat reports/mysql_lint_report.yaml >> $report_file + echo '```' >> $report_file + echo "## Oracle Linting Results" >> $report_file + echo '```yaml' >> $report_file + cat reports/oracle_lint_report.yaml >> $report_file + echo '```' >> $report_file + echo "## Syntax Check Results" >> $report_file + echo '```' >> $report_file + cat reports/syntax_report.txt >> $report_file + echo '```' >> $report_file + + - name: Upload report + uses: actions/upload-artifact@v3 + with: + name: sql-validation-report + path: reports/report-*.md + retention-days: 7 + + cleanup-old-reports: + runs-on: ubuntu-latest + permissions: + actions: write + + steps: + - name: Delete old artifacts + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const ms_per_day = 86400000; + const now = new Date(); + const sevenDaysAgo = new Date(now.getTime() - 7 * ms_per_day); + + console.log(`Fetching artifacts older than ${sevenDaysAgo.toISOString()}`); + + try { + const { data } = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + + if (data && data.artifacts && Array.isArray(data.artifacts)) { + for (const artifact of data.artifacts) { + if (new Date(artifact.created_at) < sevenDaysAgo) { + console.log(`Deleting artifact ${artifact.name} (ID: ${artifact.id})`); + await github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + }); + } + } + } else { + console.log('No artifacts found or unexpected data structure'); + } + } catch (error) { + console.error('Error occurred while processing artifacts:', error); + } diff --git a/LICENSE b/LICENSE index f9535eb..07983d5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Driver & Vehicle Standards Agency +Copyright (c) 2024 Driver & Vehicle Standards Agency Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 0e3c37b6d127b93b310fca1e4076506db5c5fb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Finbarrs=20=E1=BB=8Ck=E1=BA=B9=CC=81t=C3=BAnj=C3=AD?= <16977112+0xnu@users.noreply.github.com> Date: Mon, 21 Oct 2024 23:41:30 +0100 Subject: [PATCH 2/7] Update python-lint.yaml --- .github/workflows/python-lint.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml index e42b01f..28c82ec 100644 --- a/.github/workflows/python-lint.yaml +++ b/.github/workflows/python-lint.yaml @@ -1,10 +1,11 @@ -name: Linting - +name: Python Linting on: - push: - branches: [develop] - pull_request: - branches: [main] + workflow_call: + inputs: + python-version: + required: false + type: string + default: "3.x" jobs: lint: @@ -14,13 +15,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: ${{ inputs.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install ruff - name: Run Ruff and save results run: | + mkdir -p logs timestamp=$(date +"%Y%m%d_%H%M%S") ruff check . > logs/lint_result_${timestamp}.txt - name: Run Ruff Format From eef11db890980b364ec5f3741bcd7654f643eed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Finbarrs=20=E1=BB=8Ck=E1=BA=B9=CC=81t=C3=BAnj=C3=AD?= <16977112+0xnu@users.noreply.github.com> Date: Tue, 22 Oct 2024 00:04:14 +0100 Subject: [PATCH 3/7] Update python-lint.yaml --- .github/workflows/python-lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml index 28c82ec..c0c4d2d 100644 --- a/.github/workflows/python-lint.yaml +++ b/.github/workflows/python-lint.yaml @@ -1,4 +1,5 @@ name: Python Linting + on: workflow_call: inputs: From 8586d4e3666c301f8f5465afc990af512c586979 Mon Sep 17 00:00:00 2001 From: 0xnu Date: Sun, 23 Nov 2025 21:11:57 +0000 Subject: [PATCH 4/7] misc(feat): sql_validation Signed-off-by: 0xnu --- .github/workflows/sql-validation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sql-validation.yaml b/.github/workflows/sql-validation.yaml index 2ab9d6b..d2d4629 100644 --- a/.github/workflows/sql-validation.yaml +++ b/.github/workflows/sql-validation.yaml @@ -16,10 +16,10 @@ jobs: actions: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" From 59ff7ea3f8057c8b9e636c3b85ac7dd1288f3d60 Mon Sep 17 00:00:00 2001 From: 0xnu Date: Sun, 23 Nov 2025 21:12:59 +0000 Subject: [PATCH 5/7] misc(feat): sql_validation Signed-off-by: 0xnu --- .github/workflows/python-lint.yaml | 2 +- .github/workflows/sql-validation.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml index c0c4d2d..5164bfc 100644 --- a/.github/workflows/python-lint.yaml +++ b/.github/workflows/python-lint.yaml @@ -29,7 +29,7 @@ jobs: - name: Run Ruff Format run: ruff format --check . - name: Upload lint results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: lint-results path: logs/lint_result_*.txt diff --git a/.github/workflows/sql-validation.yaml b/.github/workflows/sql-validation.yaml index d2d4629..f7c3bf2 100644 --- a/.github/workflows/sql-validation.yaml +++ b/.github/workflows/sql-validation.yaml @@ -108,7 +108,7 @@ jobs: echo '```' >> $report_file - name: Upload report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: sql-validation-report path: reports/report-*.md From 8ba37f54f394f40b701bc2e16061e6bec43b52c6 Mon Sep 17 00:00:00 2001 From: 0xnu Date: Sun, 23 Nov 2025 21:14:33 +0000 Subject: [PATCH 6/7] misc(feat): sql_validation Signed-off-by: 0xnu --- reports/mysql_lint_report.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 reports/mysql_lint_report.yaml diff --git a/reports/mysql_lint_report.yaml b/reports/mysql_lint_report.yaml new file mode 100644 index 0000000..e69de29 From 22ade72b7493f5119fc78bf4a3ee684a2a93d014 Mon Sep 17 00:00:00 2001 From: 0xnu Date: Sun, 23 Nov 2025 21:16:35 +0000 Subject: [PATCH 7/7] misc(feat): sql_validation Signed-off-by: 0xnu --- reports/oracle_lint_report.yaml | 0 reports/syntax_report.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 reports/oracle_lint_report.yaml create mode 100644 reports/syntax_report.txt diff --git a/reports/oracle_lint_report.yaml b/reports/oracle_lint_report.yaml new file mode 100644 index 0000000..e69de29 diff --git a/reports/syntax_report.txt b/reports/syntax_report.txt new file mode 100644 index 0000000..e69de29