Skip to content

Commit 2d5a71f

Browse files
ericyangpanericyangpan
authored
Merge pull request #172 from aicodingstack/codex/catalog-automation
feat(data): automate catalog freshness
2 parents 78573fa + 20f690b commit 2d5a71f

89 files changed

Lines changed: 2476 additions & 8575 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/check-data-health.yml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
name: Check Current Data Health
2+
3+
on:
4+
schedule:
5+
- cron: '0 3 * * 1'
6+
workflow_dispatch:
7+
8+
concurrency:
9+
group: check-current-data-health
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: read
14+
issues: write
15+
16+
jobs:
17+
check-data-health:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v7
23+
24+
- name: Setup Node.js
25+
uses: actions/setup-node@v7
26+
with:
27+
node-version: '22'
28+
cache: 'npm'
29+
30+
- name: Install dependencies
31+
run: npm ci
32+
33+
- name: Check current data health
34+
id: data-health
35+
continue-on-error: true
36+
run: |
37+
npx tsx scripts/validate/data-health.ts \
38+
--as-of="$(TZ=Asia/Shanghai date +%F)" \
39+
--fail-on=error \
40+
--fail-on-code=stale-verification,future-verification-date \
41+
2>&1 | tee data-health-current.log
42+
43+
- name: Report stale or invalid data
44+
if: steps.data-health.outcome == 'failure'
45+
uses: actions/github-script@v8
46+
with:
47+
script: |
48+
const fs = require('node:fs');
49+
const title = 'Current manifest data requires review';
50+
const output = fs.readFileSync('data-health-current.log', 'utf8').slice(-6000);
51+
const body = `## Current Data Health Check Failed
52+
53+
The scheduled check evaluates freshness against today's date in Asia/Shanghai without
54+
rewriting the committed snapshot.
55+
56+
**Check run:** [View details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
57+
58+
\`\`\`text
59+
${output}
60+
\`\`\``;
61+
62+
const issues = await github.paginate(github.rest.issues.listForRepo, {
63+
owner: context.repo.owner,
64+
repo: context.repo.repo,
65+
state: 'open',
66+
labels: 'automated',
67+
per_page: 100
68+
});
69+
const existingIssue = issues.find(issue => issue.title === title);
70+
71+
if (!existingIssue) {
72+
await github.rest.issues.create({
73+
owner: context.repo.owner,
74+
repo: context.repo.repo,
75+
title,
76+
body,
77+
labels: ['automated', 'maintenance']
78+
});
79+
} else {
80+
await github.rest.issues.update({
81+
owner: context.repo.owner,
82+
repo: context.repo.repo,
83+
issue_number: existingIssue.number,
84+
body
85+
});
86+
}
87+
88+
- name: Close resolved data-health issues
89+
if: steps.data-health.outcome == 'success'
90+
uses: actions/github-script@v8
91+
with:
92+
script: |
93+
const issues = await github.paginate(github.rest.issues.listForRepo, {
94+
owner: context.repo.owner,
95+
repo: context.repo.repo,
96+
state: 'open',
97+
labels: 'automated',
98+
per_page: 100
99+
});
100+
101+
for (const issue of issues.filter(issue => issue.title === 'Current manifest data requires review')) {
102+
await github.rest.issues.createComment({
103+
owner: context.repo.owner,
104+
repo: context.repo.repo,
105+
issue_number: issue.number,
106+
body: `Current-date data health passed in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`
107+
});
108+
await github.rest.issues.update({
109+
owner: context.repo.owner,
110+
repo: context.repo.repo,
111+
issue_number: issue.number,
112+
state: 'closed',
113+
state_reason: 'completed'
114+
});
115+
}
116+
117+
- name: Fail after reporting
118+
if: steps.data-health.outcome == 'failure'
119+
run: exit 1

‎.github/workflows/ci.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,6 @@ jobs:
7373
- name: Check data health
7474
run: npm run data-health:check
7575

76-
- name: Check manifest changelog
77-
run: npm run changelog:check
78-
env:
79-
CHANGELOG_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
80-
8176
spell-check:
8277
name: Spell Check
8378
runs-on: ubuntu-latest

‎.github/workflows/monitor-model-sources.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Monitor Model Pricing and Lifecycle Sources
2+
3+
on:
4+
schedule:
5+
- cron: '0 4 * * 1'
6+
workflow_dispatch:
7+
8+
concurrency:
9+
group: monitor-model-sources
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: write
14+
pull-requests: write
15+
16+
jobs:
17+
monitor-model-sources:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v7
23+
24+
- name: Setup Node.js
25+
uses: actions/setup-node@v7
26+
with:
27+
node-version: '22'
28+
cache: 'npm'
29+
30+
- name: Install dependencies
31+
run: npm ci
32+
33+
- name: Check monitored official sources
34+
run: npm run fetch:model-sources
35+
36+
- name: Validate monitoring metadata
37+
run: |
38+
npm run test:validate
39+
npm run format:check
40+
npm run biome:check
41+
git diff --check
42+
43+
- name: Check for changes
44+
id: git-check
45+
run: |
46+
git diff --exit-code manifests/models || echo "changes=true" >> "$GITHUB_OUTPUT"
47+
48+
- name: Create review pull request
49+
if: steps.git-check.outputs.changes == 'true'
50+
uses: peter-evans/create-pull-request@v8
51+
with:
52+
token: ${{ secrets.GITHUB_TOKEN }}
53+
commit-message: 'chore(data): flag changed model sources'
54+
title: 'chore(data): review changed model sources'
55+
body: |
56+
## Review required
57+
One or more monitored official pricing or lifecycle pages changed.
58+
59+
This automation updates only normalized source-content digests and observation dates.
60+
It deliberately does **not** change token prices, model availability, lifecycle status,
61+
or any other model fact. Review the cited official pages before changing those fields.
62+
branch: auto-review-model-sources
63+
delete-branch: true
64+
labels: |
65+
automated

‎.github/workflows/update-benchmarks.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Update Benchmarks
2+
3+
on:
4+
schedule:
5+
- cron: '30 2 * * 1'
6+
workflow_dispatch:
7+
8+
concurrency:
9+
group: update-benchmarks
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: write
14+
pull-requests: write
15+
16+
jobs:
17+
update-benchmarks:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v7
23+
24+
- name: Setup Node.js
25+
uses: actions/setup-node@v7
26+
with:
27+
node-version: '22'
28+
cache: 'npm'
29+
30+
- name: Install dependencies
31+
run: npm ci
32+
33+
- name: Refresh exact benchmark mappings
34+
run: npm run fetch:benchmarks
35+
36+
- name: Regenerate manifest modules
37+
run: npm run generate:manifests
38+
39+
- name: Validate updated data
40+
run: |
41+
npm run test:validate
42+
npm run data-health:check
43+
npm run format:check
44+
npm run biome:check
45+
git diff --check
46+
47+
- name: Check for changes
48+
id: git-check
49+
run: |
50+
git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
51+
52+
- name: Create pull request
53+
if: steps.git-check.outputs.changes == 'true'
54+
uses: peter-evans/create-pull-request@v8
55+
with:
56+
token: ${{ secrets.GITHUB_TOKEN }}
57+
commit-message: 'chore(data): refresh benchmark scores'
58+
title: 'chore(data): refresh benchmark scores'
59+
body: |
60+
## Summary
61+
Automated refresh from exact entries in the official SWE-bench leaderboard data.
62+
63+
Each model manifest declares the upstream result ID and exact displayed label. The
64+
refresh fails rather than applying fuzzy model-name matching.
65+
66+
## Validation
67+
- Manifest schemas
68+
- Generated manifest modules
69+
- Data health
70+
branch: auto-update-benchmarks
71+
delete-branch: true
72+
labels: |
73+
automated

‎.github/workflows/update-product-versions.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Update Product Versions
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * 1'
6+
workflow_dispatch:
7+
8+
concurrency:
9+
group: update-product-versions
10+
cancel-in-progress: false
11+
12+
permissions:
13+
contents: write
14+
pull-requests: write
15+
16+
jobs:
17+
update-product-versions:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v7
23+
24+
- name: Setup Node.js
25+
uses: actions/setup-node@v7
26+
with:
27+
node-version: '22'
28+
cache: 'npm'
29+
30+
- name: Install dependencies
31+
run: npm ci
32+
33+
- name: Refresh tracked product versions
34+
run: npm run fetch:product-versions
35+
env:
36+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
37+
38+
- name: Regenerate manifest modules
39+
run: npm run generate:manifests
40+
41+
- name: Validate updated data
42+
run: |
43+
npm run test:validate
44+
npm run validate:i18n
45+
npm run data-health:check
46+
npm run format:check
47+
npm run biome:check
48+
git diff --check
49+
50+
- name: Check for changes
51+
id: git-check
52+
run: |
53+
git diff --exit-code manifests src/lib/generated || echo "changes=true" >> "$GITHUB_OUTPUT"
54+
55+
- name: Create pull request
56+
if: steps.git-check.outputs.changes == 'true'
57+
uses: peter-evans/create-pull-request@v8
58+
with:
59+
token: ${{ secrets.GITHUB_TOKEN }}
60+
commit-message: 'chore(data): refresh product versions'
61+
title: 'chore(data): refresh product versions'
62+
body: |
63+
## Summary
64+
Automated refresh of product versions from the registry sources declared in each manifest.
65+
66+
## Validation
67+
- Manifest schemas
68+
- Generated manifest modules
69+
- Internationalization structure
70+
- Data health
71+
72+
The workflow preserves existing values when any configured source cannot be verified.
73+
branch: auto-update-product-versions
74+
delete-branch: true
75+
labels: |
76+
automated

0 commit comments

Comments
 (0)