fix(sanity): add visitorData token to transcript sync functions and target incomplete transcripts

Author: codercatdev

apps/sanity/functions/sync-missing-youtube-transcripts/index.ts

@@ -74,17 +74,42 @@ async function fetchYouTubeCaptions(videoId: string): Promise<{
 	const INNERTUBE_CLIENT_VERSION = "20.10.38";
 
 	try {
+		// Acquire fresh visitorData token to bypass LOGIN_REQUIRED in datacenter environments
+		let visitorData: string | undefined;
+		try {
+			const pageResp = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
+				headers: {
+					"User-Agent":
+						"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+					"Accept-Language": "en-US,en;q=0.9",
+				},
+			});
+			if (pageResp.ok) {
+				const html = await pageResp.text();
+				const match = html.match(/"visitorData":"([^"]+)"/);
+				if (match) {
+					visitorData = match[1];
+				}
+			}
+		} catch {}
+
+		const requestHeaders: Record<string, string> = {
+			"Content-Type": "application/json",
+			"User-Agent": `com.google.android.youtube/${INNERTUBE_CLIENT_VERSION} (Linux; U; Android 14)`,
+		};
+		if (visitorData) {
+			requestHeaders["X-Goog-Visitor-Id"] = visitorData;
+		}
+
 		const response = await fetch(INNERTUBE_API_URL, {
 			method: "POST",
-			headers: {
-				"Content-Type": "application/json",
-				"User-Agent": `com.google.android.youtube/${INNERTUBE_CLIENT_VERSION} (Linux; U; Android 14)`,
-			},
+			headers: requestHeaders,
 			body: JSON.stringify({
 				context: {
 					client: {
 						clientName: "ANDROID",
 						clientVersion: INNERTUBE_CLIENT_VERSION,
+						...(visitorData ? { visitorData } : {}),
 					},
 				},
 				videoId,
@@ -262,24 +287,9 @@ export const handler = scheduledEventHandler(async ({ context }) => {
 	const batchSize = parseInt(process.env.SYNC_BATCH_SIZE || "50", 10);
 
 	console.log(
-		`[Sync Missing YouTube Transcripts] Querying documents with YouTube URLs missing transcripts in project ${projectId}, dataset ${dataset}...`,
+		`[Sync Missing YouTube Transcripts] Querying documents with YouTube URLs missing transcripts or full captions in project ${projectId}, dataset ${dataset}...`,
 	);
 
-	const totalMissing = await client.fetch<number>(
-		`count(*[_type in ["post", "podcast", "course", "lesson", "short"] && (defined(youtube) || defined(listenLinks.youtube)) && !defined(transcript) && !(_id in path("drafts.**"))])`,
-	);
-
-	console.log(
-		`[Sync Missing YouTube Transcripts] Total documents currently missing transcripts in ${dataset}: ${totalMissing}`,
-	);
-
-	if (totalMissing === 0) {
-		console.log(
-			"[Sync Missing YouTube Transcripts] All videos currently have linked transcripts. Done.",
-		);
-		return;
-	}
-
 	const missingDocs = await client.fetch<
 		Array<{
 			_id: string;
@@ -288,7 +298,7 @@ export const handler = scheduledEventHandler(async ({ context }) => {
 			youtube: string;
 		}>
 	>(
-		`*[_type in ["post", "podcast", "course", "lesson", "short"] && (defined(youtube) || defined(listenLinks.youtube)) && !defined(transcript) && !(_id in path("drafts.**"))][0...$batchSize]{
+		`*[_type in ["post", "podcast", "course", "lesson", "short"] && (defined(youtube) || defined(listenLinks.youtube)) && (!defined(transcript) || !defined(transcript->fullText) || length(transcript->fullText) < 50) && !(_id in path("drafts.**"))][0...$batchSize]{
 			_id,
 			_type,
 			title,

apps/sanity/functions/sync-youtube-transcript/index.ts

@@ -85,17 +85,42 @@ async function fetchYouTubeCaptions(videoId: string): Promise<{
 	const INNERTUBE_CLIENT_VERSION = "20.10.38";
 
 	try {
+		// Acquire fresh visitorData token to bypass LOGIN_REQUIRED in datacenter environments
+		let visitorData: string | undefined;
+		try {
+			const pageResp = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
+				headers: {
+					"User-Agent":
+						"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+					"Accept-Language": "en-US,en;q=0.9",
+				},
+			});
+			if (pageResp.ok) {
+				const html = await pageResp.text();
+				const match = html.match(/"visitorData":"([^"]+)"/);
+				if (match) {
+					visitorData = match[1];
+				}
+			}
+		} catch {}
+
+		const requestHeaders: Record<string, string> = {
+			"Content-Type": "application/json",
+			"User-Agent": `com.google.android.youtube/${INNERTUBE_CLIENT_VERSION} (Linux; U; Android 14)`,
+		};
+		if (visitorData) {
+			requestHeaders["X-Goog-Visitor-Id"] = visitorData;
+		}
+
 		const response = await fetch(INNERTUBE_API_URL, {
 			method: "POST",
-			headers: {
-				"Content-Type": "application/json",
-				"User-Agent": `com.google.android.youtube/${INNERTUBE_CLIENT_VERSION} (Linux; U; Android 14)`,
-			},
+			headers: requestHeaders,
 			body: JSON.stringify({
 				context: {
 					client: {
 						clientName: "ANDROID",
 						clientVersion: INNERTUBE_CLIENT_VERSION,
+						...(visitorData ? { visitorData } : {}),
 					},
 				},
 				videoId,