diff --git a/.github/workflows/python-lint.yaml b/.github/workflows/python-lint.yaml new file mode 100644 index 0000000..5164bfc --- /dev/null +++ b/.github/workflows/python-lint.yaml @@ -0,0 +1,35 @@ +name: Python Linting + +on: + workflow_call: + inputs: + python-version: + required: false + type: string + default: "3.x" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Run Ruff and save results + run: | + mkdir -p logs + timestamp=$(date +"%Y%m%d_%H%M%S") + ruff check . > logs/lint_result_${timestamp}.txt + - name: Run Ruff Format + run: ruff format --check . + - name: Upload lint results + uses: actions/upload-artifact@v4 + with: + name: lint-results + path: logs/lint_result_*.txt diff --git a/.github/workflows/sql-validation.yaml b/.github/workflows/sql-validation.yaml new file mode 100644 index 0000000..f7c3bf2 --- /dev/null +++ b/.github/workflows/sql-validation.yaml @@ -0,0 +1,156 @@ +name: Validation, Reporting, and Cleanup + +on: + push: + branches: [develop] + pull_request: + branches: [main] + schedule: + - cron: "0 2 * * 5" # Run every Friday at 2 AM UTC + +jobs: + validate-and-report: + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install sqlfluff==3.2.0 sqlparse==0.5.1 + + - name: Create reports directory + run: mkdir -p reports + + - name: Lint Oracle files + run: | + echo "Linting Oracle files:" + if [ -d "oracle" ]; then + echo "Oracle directory found. Contents:" + ls -R oracle + echo "Running SQLFluff on Oracle files:" + sqlfluff lint --dialect oracle --format yaml oracle || true + echo "Saving lint results:" + if sqlfluff lint --dialect oracle --format yaml oracle > reports/oracle_lint_report.yaml; then + echo "Oracle linting completed successfully" + else + echo "Oracle linting encountered issues, check the report for details" + fi + echo "Oracle lint report:" + cat reports/oracle_lint_report.yaml + else + echo "No Oracle directory found" + fi + + - name: Lint MySQL files + run: | + echo "Linting MySQL files:" + if [ -d "mysql" ]; then + echo "MySQL directory found. Contents:" + ls -R mysql + echo "Running SQLFluff on MySQL files:" + sqlfluff lint --dialect mysql --format yaml mysql || true + echo "Saving lint results:" + if sqlfluff lint --dialect mysql --format yaml mysql > reports/mysql_lint_report.yaml; then + echo "MySQL linting completed successfully" + else + echo "MySQL linting encountered issues, check the report for details" + fi + echo "MySQL lint report:" + cat reports/mysql_lint_report.yaml + else + echo "No MySQL directory found" + fi + + - name: Check SQL syntax + run: | + echo "Checking SQL syntax:" + touch reports/syntax_report.txt + find_output=$(find . -name "*.sql") + if [ -z "$find_output" ]; then + echo "No SQL files found" | tee -a reports/syntax_report.txt + else + while IFS= read -r file; do + echo "Checking $file" | tee -a reports/syntax_report.txt + if ! cat "$file" | sqlparse --validate >> reports/syntax_report.txt 2>&1; then + echo "Error validating $file" | tee -a reports/syntax_report.txt + fi + echo "" >> reports/syntax_report.txt + done <<< "$find_output" + fi + echo "Syntax check complete. Report contents:" + cat reports/syntax_report.txt + + - name: Generate Markdown Report + run: | + report_file="reports/report-$(date +'%Y%m%d-%H%M%S').md" + echo "# SQL Validation Report" > $report_file + echo "## MySQL Linting Results" >> $report_file + echo '```yaml' >> $report_file + cat reports/mysql_lint_report.yaml >> $report_file + echo '```' >> $report_file + echo "## Oracle Linting Results" >> $report_file + echo '```yaml' >> $report_file + cat reports/oracle_lint_report.yaml >> $report_file + echo '```' >> $report_file + echo "## Syntax Check Results" >> $report_file + echo '```' >> $report_file + cat reports/syntax_report.txt >> $report_file + echo '```' >> $report_file + + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: sql-validation-report + path: reports/report-*.md + retention-days: 7 + + cleanup-old-reports: + runs-on: ubuntu-latest + permissions: + actions: write + + steps: + - name: Delete old artifacts + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const ms_per_day = 86400000; + const now = new Date(); + const sevenDaysAgo = new Date(now.getTime() - 7 * ms_per_day); + + console.log(`Fetching artifacts older than ${sevenDaysAgo.toISOString()}`); + + try { + const { data } = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + + if (data && data.artifacts && Array.isArray(data.artifacts)) { + for (const artifact of data.artifacts) { + if (new Date(artifact.created_at) < sevenDaysAgo) { + console.log(`Deleting artifact ${artifact.name} (ID: ${artifact.id})`); + await github.rest.actions.deleteArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + }); + } + } + } else { + console.log('No artifacts found or unexpected data structure'); + } + } catch (error) { + console.error('Error occurred while processing artifacts:', error); + } diff --git a/LICENSE b/LICENSE index f9535eb..07983d5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Driver & Vehicle Standards Agency +Copyright (c) 2024 Driver & Vehicle Standards Agency Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/reports/mysql_lint_report.yaml b/reports/mysql_lint_report.yaml new file mode 100644 index 0000000..e69de29 diff --git a/reports/oracle_lint_report.yaml b/reports/oracle_lint_report.yaml new file mode 100644 index 0000000..e69de29 diff --git a/reports/syntax_report.txt b/reports/syntax_report.txt new file mode 100644 index 0000000..e69de29