Skip to content

Commit bad84af

Browse files
codercatdevcodercatdev
committed
fix(ci): use HTTP export and mutate API with SANITY_API_WRITE_TOKEN to sync production to dev dataset
1 parent a1412b4 commit bad84af

1 file changed

Lines changed: 100 additions & 11 deletions

File tree

‎.github/workflows/nightly-dataset-sync.yml‎

Lines changed: 100 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,28 +27,117 @@ jobs:
2727
env:
2828
SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID || 'hfh83o0w' }}
2929
SANITY_AUTH_TOKEN: ${{ secrets.SANITY_AUTH_TOKEN }}
30+
SANITY_API_WRITE_TOKEN: ${{ secrets.SANITY_API_WRITE_TOKEN }}
3031
run: |
3132
PROJECT_ID="${SANITY_STUDIO_PROJECT_ID:-hfh83o0w}"
3233
API_HOST="https://api.sanity.io"
3334
API_VERSION="v2021-06-07"
3435
35-
echo "Ensuring 'dev' dataset exists (public ACL)..."
36-
curl -s -X PUT "$API_HOST/$API_VERSION/projects/$PROJECT_ID/datasets/dev" \
37-
-H "Authorization: Bearer $SANITY_AUTH_TOKEN" \
38-
-H "Content-Type: application/json" \
39-
-d '{"aclMode":"public"}' || true
36+
echo "1. Ensuring 'dev' dataset exists (public ACL)..."
37+
for T in "$SANITY_AUTH_TOKEN" "$SANITY_API_WRITE_TOKEN"; do
38+
[ -z "$T" ] && continue
39+
RESP=$(curl -s -X PUT "$API_HOST/$API_VERSION/projects/$PROJECT_ID/datasets/dev" \
40+
-H "Authorization: Bearer $T" \
41+
-H "Content-Type: application/json" \
42+
-d '{"aclMode":"public"}' || true)
43+
echo "Dataset create response: $RESP"
44+
if echo "$RESP" | grep -q '"datasetName":"dev"'; then
45+
break
46+
fi
47+
done
4048
41-
echo "Exporting 'production' dataset..."
42-
pnpm --filter @codingcatdev/sanity exec sanity dataset export production /tmp/production.tar.gz --overwrite
49+
echo "2. Exporting 'production' dataset to /tmp/production.ndjson..."
50+
curl -f -sS "https://$PROJECT_ID.api.sanity.io/$API_VERSION/data/export/production" -o /tmp/production.ndjson
51+
gzip -k -f /tmp/production.ndjson
52+
wc -l /tmp/production.ndjson
4353
44-
echo "Importing into 'dev' dataset (--replace)..."
45-
pnpm --filter @codingcatdev/sanity exec sanity dataset import /tmp/production.tar.gz dev --replace
54+
echo "3. Importing documents into 'dev' dataset via Sanity Mutate API..."
55+
node -e '
56+
const fs = require("fs");
57+
const projectId = process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w";
58+
const tokens = [process.env.SANITY_API_WRITE_TOKEN, process.env.SANITY_AUTH_TOKEN].filter(Boolean);
59+
const lines = fs.readFileSync("/tmp/production.ndjson", "utf8").trim().split("\n").filter(Boolean);
60+
const allDocs = lines.map((l) => JSON.parse(l));
61+
// Exclude sanity.imageAsset and sanity.fileAsset if mutate API rejects system asset docs without binary upload,
62+
// or include content docs and weaken references to assets/missing docs so all content imports cleanly.
63+
const contentDocs = allDocs.filter((d) => d._type !== "sanity.imageAsset" && d._type !== "sanity.fileAsset");
64+
const contentIds = new Set(contentDocs.map((d) => d._id));
4665
47-
echo "Dataset sync completed successfully!"
66+
function sanitizeDoc(obj) {
67+
if (!obj || typeof obj !== "object") return obj;
68+
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
69+
const copy = {};
70+
for (const [k, v] of Object.entries(obj)) {
71+
if (k === "_rev") continue;
72+
copy[k] = sanitizeDoc(v);
73+
}
74+
if (typeof copy._ref === "string" && !contentIds.has(copy._ref)) {
75+
copy._weak = true;
76+
}
77+
return copy;
78+
}
79+
80+
const sanitized = contentDocs.map(sanitizeDoc);
81+
console.log(`Prepared ${sanitized.length} content documents (from ${allDocs.length} total lines) for dataset "dev"...`);
82+
83+
async function mutateBatch(mutations) {
84+
let lastErr = "";
85+
for (const token of tokens) {
86+
const res = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/data/mutate/dev?visibility=async&autoGenerateArrayKeys=true`, {
87+
method: "POST",
88+
headers: {
89+
Authorization: `Bearer ${token}`,
90+
"Content-Type": "application/json",
91+
},
92+
body: JSON.stringify({ mutations }),
93+
});
94+
if (res.ok) return;
95+
lastErr = `HTTP ${res.status}: ${await res.text()}`;
96+
}
97+
throw new Error(`Mutate batch failed: ${lastErr}`);
98+
}
99+
100+
(async () => {
101+
// Pass 1: Create stubs so all internal document references resolve
102+
const stubs = sanitized.map((d) => ({ createOrReplace: { _id: d._id, _type: d._type } }));
103+
for (let i = 0; i < stubs.length; i += 250) {
104+
await mutateBatch(stubs.slice(i, i + 250));
105+
}
106+
console.log("Pass 1 (document stubs) complete.");
107+
108+
// Pass 2: Populate full document bodies
109+
for (let i = 0; i < sanitized.length; i += 100) {
110+
const batch = sanitized.slice(i, i + 100).map((d) => ({ createOrReplace: d }));
111+
await mutateBatch(batch);
112+
if ((i / 100) % 5 === 0) {
113+
console.log(`Imported ${Math.min(i + 100, sanitized.length)} / ${sanitized.length} documents...`);
114+
}
115+
}
116+
117+
// Pass 3 (best-effort): Import asset metadata documents
118+
const assetDocs = allDocs
119+
.filter((d) => d._type === "sanity.imageAsset" || d._type === "sanity.fileAsset")
120+
.map(({ _rev, ...rest }) => rest);
121+
try {
122+
for (let i = 0; i < assetDocs.length; i += 250) {
123+
const batch = assetDocs.slice(i, i + 250).map((d) => ({ createOrReplace: d }));
124+
await mutateBatch(batch);
125+
}
126+
console.log(`Imported ${assetDocs.length} asset metadata documents.`);
127+
} catch (assetErr) {
128+
console.warn("Note: Asset metadata docs skipped (served via CDN):", assetErr.message);
129+
}
130+
131+
console.log("Dataset sync completed successfully!");
132+
})().catch((err) => {
133+
console.error(err);
134+
process.exit(1);
135+
});
136+
'
48137
49138
- name: Upload Sanity production dataset backup artifact
50139
uses: actions/upload-artifact@v4
51140
with:
52141
name: sanity-production-backup-${{ github.run_id }}
53-
path: /tmp/production.tar.gz
142+
path: /tmp/production.ndjson.gz
54143
retention-days: 30

0 commit comments

Comments
 (0)