Skip to content

Commit 7c7384d

Browse files
codercatdevcodercatdev
committed
fix(ci): use heredoc in nightly-dataset-sync.yml to avoid bash single-quote escaping
1 parent a06ffa2 commit 7c7384d

1 file changed

Lines changed: 88 additions & 94 deletions

File tree

‎.github/workflows/nightly-dataset-sync.yml‎

Lines changed: 88 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -52,105 +52,99 @@ jobs:
5252
wc -l /tmp/production.ndjson
5353
5454
echo "3. Importing documents into 'dev' dataset via Sanity Mutate API..."
55-
node -e '
56-
const fs = require("fs");
57-
const projectId = process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w";
58-
const tokens = [process.env.SANITY_API_WRITE_TOKEN, process.env.SANITY_AUTH_TOKEN].filter(Boolean);
59-
const lines = fs.readFileSync("/tmp/production.ndjson", "utf8").trim().split("\n").filter(Boolean);
60-
const allDocs = lines.map((l) => JSON.parse(l));
61-
// Exclude sanity.imageAsset and sanity.fileAsset if mutate API rejects system asset docs without binary upload,
62-
// or include content docs and weaken references to assets/missing docs so all content imports cleanly.
63-
const contentDocs = allDocs.filter((d) => d._type !== "sanity.imageAsset" && d._type !== "sanity.fileAsset");
64-
const contentIds = new Set(contentDocs.map((d) => d._id));
65-
66-
function sanitizeDoc(obj) {
67-
if (!obj || typeof obj !== "object") return obj;
68-
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
69-
const copy = {};
70-
for (const [k, v] of Object.entries(obj)) {
71-
if (k === "_rev") continue;
72-
copy[k] = sanitizeDoc(v);
73-
}
74-
if (typeof copy._ref === "string" && !contentIds.has(copy._ref)) {
75-
copy._weak = true;
76-
}
77-
return copy;
55+
cat << 'EOF' > /tmp/sync.cjs
56+
const fs = require("fs");
57+
const projectId = process.env.SANITY_STUDIO_PROJECT_ID || "hfh83o0w";
58+
const tokens = [process.env.SANITY_API_WRITE_TOKEN, process.env.SANITY_AUTH_TOKEN].filter(Boolean);
59+
const lines = fs.readFileSync("/tmp/production.ndjson", "utf8").trim().split("\n").filter(Boolean);
60+
const allDocs = lines.map((l) => JSON.parse(l));
61+
const contentDocs = allDocs.filter((d) => d._type !== "sanity.imageAsset" && d._type !== "sanity.fileAsset");
62+
const contentIds = new Set(contentDocs.map((d) => d._id));
63+
64+
function sanitizeDoc(obj) {
65+
if (!obj || typeof obj !== "object") return obj;
66+
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
67+
const copy = {};
68+
for (const [k, v] of Object.entries(obj)) {
69+
if (k === "_rev") continue;
70+
copy[k] = sanitizeDoc(v);
7871
}
79-
80-
const sanitized = contentDocs.map(sanitizeDoc);
81-
console.log(`Prepared ${sanitized.length} content documents (from ${allDocs.length} total lines) for dataset "dev"...`);
82-
83-
async function mutateBatch(mutations) {
84-
let lastErr = "";
85-
for (const token of tokens) {
86-
const res = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/data/mutate/dev?visibility=async&autoGenerateArrayKeys=true`, {
87-
method: "POST",
88-
headers: {
89-
Authorization: `Bearer ${token}`,
90-
"Content-Type": "application/json",
91-
},
92-
body: JSON.stringify({ mutations }),
93-
});
94-
if (res.ok) return;
95-
lastErr = `HTTP ${res.status}: ${await res.text()}`;
96-
}
97-
throw new Error(`Mutate batch failed: ${lastErr}`);
72+
if (typeof copy._ref === "string" && !contentIds.has(copy._ref)) {
73+
copy._weak = true;
9874
}
99-
100-
(async () => {
101-
// Helper to chunk mutations by both count and byte size (< 1.5MB, well below Sanity's 4MB limit)
102-
async function sendChunked(items, label) {
103-
let batch = [];
104-
let batchBytes = 0;
105-
let sent = 0;
106-
const MAX_ITEMS = 50;
107-
const MAX_BYTES = 1500000;
108-
109-
for (const item of items) {
110-
const mutation = { createOrReplace: item };
111-
const itemBytes = Buffer.byteLength(JSON.stringify(mutation), "utf8");
112-
if (batch.length > 0 && (batch.length >= MAX_ITEMS || batchBytes + itemBytes > MAX_BYTES)) {
113-
await mutateBatch(batch);
114-
sent += batch.length;
115-
if (sent % 500 < batch.length) {
116-
console.log(`[${label}] Imported ${sent} / ${items.length}...`);
117-
}
118-
batch = [];
119-
batchBytes = 0;
120-
}
121-
batch.push(mutation);
122-
batchBytes += itemBytes;
123-
}
124-
if (batch.length > 0) {
125-
await mutateBatch(batch);
126-
sent += batch.length;
75+
return copy;
76+
}
77+
78+
const sanitized = contentDocs.map(sanitizeDoc);
79+
console.log(`Prepared ${sanitized.length} content documents (from ${allDocs.length} total lines) for dataset "dev"...`);
80+
81+
async function mutateBatch(mutations) {
82+
let lastErr = "";
83+
for (const token of tokens) {
84+
const res = await fetch(`https://${projectId}.api.sanity.io/v2021-06-07/data/mutate/dev?visibility=async&autoGenerateArrayKeys=true`, {
85+
method: "POST",
86+
headers: {
87+
Authorization: `Bearer ${token}`,
88+
"Content-Type": "application/json",
89+
},
90+
body: JSON.stringify({ mutations }),
91+
});
92+
if (res.ok) return;
93+
lastErr = `HTTP ${res.status}: ${await res.text()}`;
94+
}
95+
throw new Error(`Mutate batch failed: ${lastErr}`);
96+
}
97+
98+
async function sendChunked(items, label) {
99+
let batch = [];
100+
let batchBytes = 0;
101+
let sent = 0;
102+
const MAX_ITEMS = 50;
103+
const MAX_BYTES = 1500000;
104+
105+
for (const item of items) {
106+
const mutation = { createOrReplace: item };
107+
const itemBytes = Buffer.byteLength(JSON.stringify(mutation), "utf8");
108+
if (batch.length > 0 && (batch.length >= MAX_ITEMS || batchBytes + itemBytes > MAX_BYTES)) {
109+
await mutateBatch(batch);
110+
sent += batch.length;
111+
if (sent % 500 < batch.length) {
112+
console.log(`[${label}] Imported ${sent} / ${items.length}...`);
127113
}
128-
console.log(`[${label}] Completed (${sent} / ${items.length}).`);
129-
}
130-
131-
// Pass 1: Create stubs so all internal document references resolve
132-
const stubs = sanitized.map((d) => ({ _id: d._id, _type: d._type }));
133-
await sendChunked(stubs, "Pass 1: Stubs");
134-
135-
// Pass 2: Populate full document bodies
136-
await sendChunked(sanitized, "Pass 2: Content Docs");
137-
138-
// Pass 3 (best-effort): Import asset metadata documents so Studio Media Library resolves in dev
139-
const assetDocs = allDocs
140-
.filter((d) => d._type === "sanity.imageAsset" || d._type === "sanity.fileAsset")
141-
.map(({ _rev, ...rest }) => rest);
142-
try {
143-
await sendChunked(assetDocs, "Pass 3: Asset Docs");
144-
} catch (assetErr) {
145-
console.warn("Note: Asset metadata docs skipped (served via CDN):", assetErr.message);
114+
batch = [];
115+
batchBytes = 0;
146116
}
117+
batch.push(mutation);
118+
batchBytes += itemBytes;
119+
}
120+
if (batch.length > 0) {
121+
await mutateBatch(batch);
122+
sent += batch.length;
123+
}
124+
console.log(`[${label}] Completed (${sent} / ${items.length}).`);
125+
}
126+
127+
(async () => {
128+
const stubs = sanitized.map((d) => ({ _id: d._id, _type: d._type }));
129+
await sendChunked(stubs, "Pass 1: Stubs");
130+
await sendChunked(sanitized, "Pass 2: Content Docs");
131+
132+
const assetDocs = allDocs
133+
.filter((d) => d._type === "sanity.imageAsset" || d._type === "sanity.fileAsset")
134+
.map(({ _rev, ...rest }) => rest);
135+
try {
136+
await sendChunked(assetDocs, "Pass 3: Asset Docs");
137+
} catch (assetErr) {
138+
console.warn("Note: Asset metadata docs skipped (served via CDN):", assetErr.message);
139+
}
147140
148-
console.log("Dataset sync completed successfully!");
149-
})().catch((err) => {
150-
console.error(err);
151-
process.exit(1);
152-
});
153-
'
141+
console.log("Dataset sync completed successfully!");
142+
})().catch((err) => {
143+
console.error(err);
144+
process.exit(1);
145+
});
146+
EOF
147+
node /tmp/sync.cjs
154148
155149
- name: Upload Sanity production dataset backup artifact
156150
uses: actions/upload-artifact@v4

0 commit comments

Comments
 (0)