From d62e6d1488a636510addfb786e095ae31d24b6bb Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 1 Apr 2026 10:36:28 -0400 Subject: [PATCH 01/61] fix: increase token limit sent to LLM --- .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 197 ++++++++++++++---- .../utils/filenameTokenizer.ts | 116 ++++++++++- .../DatasetOrganizer/utils/llmPrompts.ts | 6 +- 3 files changed, 265 insertions(+), 54 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index d9d6366..a8fd052 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -184,6 +184,7 @@ const LLMPanel: React.FC = ({ }); setEvidenceBundle(bundle); + setSubjectAnalysis(null); // ← add this line downloadJSON(bundle, "evidence_bundle.json"); setStatus("✓ Evidence bundle generated and downloaded!"); } catch (err: any) { @@ -380,6 +381,43 @@ const LLMPanel: React.FC = ({ setStatus("3/3 Generating participants.tsv..."); const partsPrompt = getParticipantsPrompt(userText); + // ← ADD HERE: compute subject analysis before try block so it's in scope + const currentSubjectAnalysis = extractSubjectAnalysis( + evidenceBundle?.all_files || [], + evidenceBundle?.user_hints?.n_subjects, + evidenceBundle?.filename_analysis?.python_statistics + ?.dominant_prefixes + ); + + console.log("=== PARTICIPANTS DEBUG ==="); + console.log("method:", currentSubjectAnalysis?.method); + console.log("subject_count:", currentSubjectAnalysis?.subject_count); + console.log( + "id_mapping:", + currentSubjectAnalysis?.id_mapping?.id_mapping + ); + console.log( + "reverse_mapping:", + currentSubjectAnalysis?.id_mapping?.reverse_mapping + ); + console.log( + "subject_records sample:", + currentSubjectAnalysis?.subject_records?.slice(0, 3) + ); + const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + const expectedCount = evidenceBundle?.user_hints?.n_subjects; + const subjectLabels: string[] = + idMap && + Object.keys(idMap).length > 0 && + (!expectedCount || Object.keys(idMap).length === expectedCount) + ? Object.values(idMap).map((id: string) => `sub-${id}`) + : Array.from( + { + length: expectedCount || Object.keys(idMap || {}).length || 1, + }, + (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + ); + let partsResponse; if (currentProvider.isAnthropic) { partsResponse = await fetch(currentProvider.baseUrl, { @@ -435,52 +473,130 @@ const LLMPanel: React.FC = ({ : partsData.choices[0].message.content; // Build TSV from schema + // try { + // const schemaText = participantsRaw + // .replace(/^```json\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // const schema = JSON.parse(schemaText); + // const columns: string[] = schema.columns.map((c: any) => c.name); + + // // Get subject IDs from evidence bundle (extracted by Python-style analysis) + // // const idMapping = + // // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; + // // const subjectLabels: string[] = idMapping + // // ? Object.values(idMapping).map((id) => `sub-${id}`) + // // : ["sub-01"]; // fallback if no subject analysis + // // Get subject IDs from subjectAnalysis state (computed at plan stage) + // // Fall back to computing fresh if plan hasn't been run yet + // const currentSubjectAnalysis = + // subjectAnalysis || + // extractSubjectAnalysis( + // evidenceBundle?.all_files || [], + // evidenceBundle?.user_hints?.n_subjects, + // evidenceBundle?.filename_analysis?.python_statistics + // ?.dominant_prefixes + // ); + // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + // const subjectLabels: string[] = + // idMap && Object.keys(idMap).length > 0 + // ? Object.values(idMap).map((id) => `sub-${id}`) + // : Array.from( + // { length: evidenceBundle?.user_hints?.n_subjects || 1 }, + // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + // ); + + // const header = columns.join("\t"); + // // ====origin==== + // // const rows = subjectLabels.map((subId) => + // // columns + // // .map((col: string) => (col === "participant_id" ? subId : "n/a")) + // // .join("\t") + // // ); + // //====== end ====== + // // =====update start===== + // const reverseMap = + // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // const subjectRecords = currentSubjectAnalysis?.subject_records || []; + + // const rows = subjectLabels.map((subId) => { + // const bareId = subId.replace(/^sub-/, ""); + // const originalId = reverseMap[bareId]; + // const record = subjectRecords.find( + // (r: any) => r.original_id === originalId + // ); + // return columns + // .map((col: string) => { + // if (col === "participant_id") return subId; + // if (col === "original_id") return originalId || "n/a"; + // if (col === "group") return (record as any)?.group || "n/a"; + // return "n/a"; + // }) + // .join("\t"); + // }); + // //====update end====== + // participantsContent = [header, ...rows].join("\n"); + // } catch (e) { + // // Fallback: LLM didn't return valid JSON schema, use raw content + // participantsContent = participantsRaw + // .replace(/^```\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // } + // Build TSV from schema + subject analysis + // Mirrors _generate_participants_tsv_from_python() in planner.py try { const schemaText = participantsRaw .replace(/^```json\n?/g, "") .replace(/\n?```$/g, "") .trim(); const schema = JSON.parse(schemaText); - const columns: string[] = schema.columns.map((c: any) => c.name); - - // Get subject IDs from evidence bundle (extracted by Python-style analysis) - // const idMapping = - // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; - // const subjectLabels: string[] = idMapping - // ? Object.values(idMapping).map((id) => `sub-${id}`) - // : ["sub-01"]; // fallback if no subject analysis - // Get subject IDs from subjectAnalysis state (computed at plan stage) - // Fall back to computing fresh if plan hasn't been run yet - const currentSubjectAnalysis = - subjectAnalysis || - extractSubjectAnalysis( - evidenceBundle?.all_files || [], - evidenceBundle?.user_hints?.n_subjects, - evidenceBundle?.filename_analysis?.python_statistics - ?.dominant_prefixes + + // LLM decides extra demographic columns (sex, age, group etc.) + // but we always add participant_id and original_id ourselves + const extraColumns: string[] = schema.columns + .map((c: any) => c.name) + .filter( + (name: string) => + name !== "participant_id" && name !== "original_id" ); - const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; - const subjectLabels: string[] = - idMap && Object.keys(idMap).length > 0 - ? Object.values(idMap).map((id) => `sub-${id}`) - : Array.from( - { length: evidenceBundle?.user_hints?.n_subjects || 1 }, - (_, i) => `sub-${String(i + 1).padStart(2, "0")}` - ); + + // Always start with participant_id and original_id + const columns = ["participant_id", "original_id", ...extraColumns]; + + const reverseMap = + currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + const subjectRecords = currentSubjectAnalysis?.subject_records || []; const header = columns.join("\t"); - const rows = subjectLabels.map((subId) => - columns - .map((col: string) => (col === "participant_id" ? subId : "n/a")) - .join("\t") - ); + const rows = subjectLabels.map((subId) => { + const bareId = subId.replace(/^sub-/, ""); + const originalId = reverseMap[bareId] || "n/a"; + const record = subjectRecords.find( + (r: any) => r.original_id === originalId + ); + return columns + .map((col: string) => { + if (col === "participant_id") return subId; + if (col === "original_id") return originalId; + if (col === "group") return (record as any)?.group || "n/a"; + return "n/a"; + }) + .join("\t"); + }); + participantsContent = [header, ...rows].join("\n"); } catch (e) { - // Fallback: LLM didn't return valid JSON schema, use raw content - participantsContent = participantsRaw - .replace(/^```\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); + // Fallback: generate minimal TSV directly from subject analysis + const reverseMap = + currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + const header = "participant_id\toriginal_id"; + const rows = subjectLabels.map((subId) => { + const bareId = subId.replace(/^sub-/, ""); + const originalId = reverseMap[bareId] || "n/a"; + return `${subId}\t${originalId}`; + }); + participantsContent = [header, ...rows].join("\n"); } } // ========================================== @@ -632,11 +748,6 @@ const LLMPanel: React.FC = ({ const filePatterns = analyzeFilePatterns(files); const userContext = getUserContext(files); const annotations = getFileAnnotations(files); - // console.log("=== PROMPT BEING SENT TO LLM ==="); - // console.log(fileSummary); - // console.log(filePatterns); - // console.log(userContext); - // console.log("================================="); // UPDATED: Improved prompt that uses trio files const prompt = getConversionScriptPrompt( @@ -788,6 +899,7 @@ const LLMPanel: React.FC = ({ userNSubjects, dominantPrefixes ); + setSubjectAnalysis(computedSubjectAnalysis); const fileSummary = buildFileSummary(files); @@ -801,11 +913,6 @@ const LLMPanel: React.FC = ({ .map((s: any) => ` - ${s.relpath}`) .join("\n") || ""; - // console.log("=== SAMPLE FILES ==="); - // console.log(sampleFiles); - // console.log("=== COUNTS BY EXT ==="); - // console.log(evidenceBundle?.counts_by_ext); - const prompt = getBIDSPlanPrompt( fileSummary, filePatterns, diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts index 1c1820b..4708c13 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts +++ b/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts @@ -7,6 +7,7 @@ export interface SubjectRecord { site: string | null; pattern_name: string; file_count: number; + group?: string; } export interface SubjectAnalysis { @@ -359,14 +360,52 @@ const extractNumericIdFromIdentifier = (identifier: string): string | null => { // ── Step 1: Directory structure patterns // Mirrors _extract_subjects_from_directory_structure() in planner.py +const SKIP_DIRS = new Set([ + "anat", + "func", + "dwi", + "fmap", + "nirs", + "meg", + "eeg", + "beh", + "perf", + "derivatives", + "sourcedata", + "stimuli", + "walking", + "resting", + "resting_state", + "run", + "ses", + "pd", + "control", + "hc", + "task", + "sub", + "dataset", + "data", + "raw", + "bids", + "output", + "outputs", + "staging", + "_staging", + "mri", + "fnirs", + "edf", + "dicom", +]); + const extractFromDirectoryStructure = ( allFiles: string[] ): Omit | null => { const patterns: Array<[RegExp, boolean, number, number | null, string]> = [ - [/^([A-Za-z]+)_sub(\d+)$/i, true, 2, 1, "site_prefixed"], - [/^sub-(\w+)$/, false, 1, null, "standard_bids"], // directory named sub-01 - [/^subject[_-]?(\d+)$/i, false, 1, null, "simple"], - [/^\d{3,}$/, false, 1, null, "numeric_only"], // directory named 001 + [/^([A-Za-z]+)_sub(\d+)$/i, true, 2, 1, "site_prefixed"], // Beijing_sub82352 + [/^sub-(\w+)$/, false, 1, null, "standard_bids"], // sub-01 + [/^subject[_-]?(\d+)$/i, false, 1, null, "simple"], // subject_01 + [/^\d{3,}$/, false, 1, null, "numeric_only"], // 001 + [/^([A-Za-z]+\d+)$/, false, 1, null, "alphanum_id"], // PD01, Control01, HC03 ]; const subjectRecords: SubjectRecord[] = []; @@ -374,11 +413,14 @@ const extractFromDirectoryStructure = ( for (const filepath of allFiles) { const parts = filepath.split("/"); - // Only check the first 2 path parts (directory levels), not the filename - // mirrors: for part in parts[:2] - const dirsOnly = parts.slice(0, Math.min(2, parts.length - 1)); // exclude filename + // Check ALL directory levels (not just first 2) + const dirsOnly = parts.slice(0, parts.length - 1); + // const dirsOnly = parts.slice(0, Math.min(2, parts.length - 1)); // only first 2 levels for (const part of dirsOnly) { + // Skip known non-subject directory names + // if (SKIP_DIRS.has(part.toLowerCase())) continue; + for (const [ regex, hasSite, @@ -393,7 +435,7 @@ const extractFromDirectoryStructure = ( seenIds.add(originalId); subjectRecords.push({ original_id: originalId, - numeric_id: match[idGroup], + numeric_id: match[idGroup] || match[0], site: hasSite && siteGroup ? match[siteGroup] : null, pattern_name: patternName, file_count: 0, @@ -407,11 +449,41 @@ const extractFromDirectoryStructure = ( if (subjectRecords.length === 0) return null; subjectRecords.sort((a, b) => { + // const na = parseInt(a.numeric_id) || 0; + // const nb = parseInt(b.numeric_id) || 0; + // return na - nb; + const aMatch = a.original_id.match(/^([A-Za-z]+)(\d+)$/); + const bMatch = b.original_id.match(/^([A-Za-z]+)(\d+)$/); + + if (aMatch && bMatch) { + const prefixCompare = aMatch[1].localeCompare(bMatch[1]); + if (prefixCompare !== 0) return prefixCompare; + return parseInt(aMatch[2]) - parseInt(bMatch[2]); + } + const na = parseInt(a.numeric_id) || 0; const nb = parseInt(b.numeric_id) || 0; return na - nb; }); + // Build group map: subject originalId → parent directory name + // const groupMap: Record = {}; + // for (const filepath of allFiles) { + // const parts = filepath.split("/"); + // for (let i = 1; i < parts.length - 1; i++) { + // if (seenIds.has(parts[i]) && !SKIP_DIRS.has(parts[i - 1].toLowerCase())) { + // groupMap[parts[i]] = parts[i - 1]; + // } + // } + // } + + // // Attach group to each record + // for (const rec of subjectRecords) { + // if (groupMap[rec.original_id]) { + // rec.group = groupMap[rec.original_id]; + // } + // } + return { success: true, method: "directory_structure", @@ -551,6 +623,34 @@ export const extractSubjectAnalysis = ( python_generated_filename_rules: [], }; } + // bug fix for subject mapping + // === original + // const idMapping = generateIdMapping(subjectInfo); + // return { ...subjectInfo, id_mapping: idMapping }; + // ==== end + // ==== updates + // CRITICAL: n_subjects is authoritative (mirrors planner.py PROMPT_BIDS_PLAN) + // If analysis count doesn't match user input, fall back to sequential numbering + const expectedCount = userNSubjects; + if (expectedCount && subjectInfo.subject_count !== expectedCount) { + const idMap: Record = {}; + const reverseMap: Record = {}; + for (let i = 1; i <= expectedCount; i++) { + const bidsId = String(i).padStart(2, "0"); + idMap[`sub-${bidsId}`] = bidsId; + reverseMap[bidsId] = `sub-${bidsId}`; + } + return { + ...subjectInfo, + subject_count: expectedCount, + id_mapping: { + id_mapping: idMap, + reverse_mapping: reverseMap, + strategy_used: "numeric_fallback", + metadata_columns: [], + }, + }; + } const idMapping = generateIdMapping(subjectInfo); return { ...subjectInfo, id_mapping: idMapping }; diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/llmPrompts.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/llmPrompts.ts index 6d6a796..be03f83 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/utils/llmPrompts.ts +++ b/src/components/User/Dashboard/DatasetOrganizer/utils/llmPrompts.ts @@ -9,9 +9,13 @@ export const getDatasetDescriptionPrompt = ( evidenceBundle?: any ): string => { const documentsContext = + // evidenceBundle?.documents + // ?.map((d: any) => `[${d.filename}]:\n${d.content}`) + // .join("\n\n") || ""; evidenceBundle?.documents - ?.map((d: any) => `[${d.filename}]:\n${d.content}`) + ?.map((d: any) => `[${d.filename}]:\n${(d.content || "").slice(0, 500)}`) .join("\n\n") || ""; + return `You are a BIDS dataset_description.json generator. CRITICAL: Use the following user-provided content to extract dataset information! From 843f0a21d95bc273bb2374f57647a6b6bc69cd95 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 1 Apr 2026 15:05:56 -0400 Subject: [PATCH 02/61] fix: resolve leaks in previewModal 3D viewer --- src/components/PreviewModal.tsx | 36 +- src/utils/preview.js | 600 +++++++++++++++++++++++--------- 2 files changed, 454 insertions(+), 182 deletions(-) diff --git a/src/components/PreviewModal.tsx b/src/components/PreviewModal.tsx index 4a3ce82..bf81318 100644 --- a/src/components/PreviewModal.tsx +++ b/src/components/PreviewModal.tsx @@ -28,30 +28,14 @@ const PreviewModal: React.FC<{ // fix end--------------------- useEffect(() => { - if (!isOpen) return; - //add spinner - // if (!isOpen || isLoading) return; - - // fix start-----------: Get the container element from the ref. - // const container = canvasContainerRef.current; - // if (!container) { - // // This can happen briefly on the first render, so we just wait for the next render. - // return; - // } - // // 3. Check for the required legacy functions on the window object. - // if ( - // typeof window.previewdata !== "function" || - // typeof window.initcanvas_with_container !== "function" - // ) { - // console.error( - // "❌ Legacy preview script functions are not available on the window object." - // ); - // return; - // } - - // window.previewdata(dataKey, previewIndex, isInternal, false); - // fix end--------------------------------- - // clear old canvas + // if (!isOpen) return; + if (!isOpen) { + // Modal just closed — clean up Three.js immediately + if (typeof window.destroyPreview === "function") { + window.destroyPreview(); + } + return; + } const canvasDiv = document.getElementById("canvas"); if (canvasDiv) while (canvasDiv.firstChild) canvasDiv.removeChild(canvasDiv.firstChild); @@ -69,6 +53,10 @@ const PreviewModal: React.FC<{ return () => { clearInterval(interval); + // Component unmounting — clean up Three.js + if (typeof window.destroyPreview === "function") { + window.destroyPreview(); + } }; }, [isOpen, dataKey, previewIndex, isInternal]); diff --git a/src/utils/preview.js b/src/utils/preview.js index 912c8f8..496e855 100644 --- a/src/utils/preview.js +++ b/src/utils/preview.js @@ -74,18 +74,55 @@ var typedfun = { BigUint64Array: null, }; +// function destroyPreview() { +// if (window.scene) { +// while (window.scene.children.length > 0) { +// const obj = window.scene.children[0]; +// window.scene.remove(obj); +// if (obj.geometry) obj.geometry.dispose(); +// if (obj.material) obj.material.dispose(); +// } +// } + +// if (window.renderer && window.renderer.domElement) { +// window.renderer.domElement.remove(); +// window.renderer.dispose(); +// } + +// window.scene = undefined; +// window.camera = undefined; +// window.renderer = undefined; +// window.controls = undefined; +// window.reqid = undefined; +// } function destroyPreview() { + // Cancel animation loop first + if (reqid !== undefined) { + cancelAnimationFrame(reqid); + reqid = undefined; + window.reqid = undefined; + } + if (window.scene) { while (window.scene.children.length > 0) { const obj = window.scene.children[0]; window.scene.remove(obj); if (obj.geometry) obj.geometry.dispose(); - if (obj.material) obj.material.dispose(); + if (obj.material) { + // Dispose any textures stored in shader uniforms + if (obj.material.uniforms) { + Object.values(obj.material.uniforms).forEach((u) => { + if (u.value && u.value.isTexture) u.value.dispose(); + }); + } + obj.material.dispose(); + } } } if (window.renderer && window.renderer.domElement) { window.renderer.domElement.remove(); + window.renderer.forceContextLoss(); window.renderer.dispose(); } @@ -93,7 +130,9 @@ function destroyPreview() { window.camera = undefined; window.renderer = undefined; window.controls = undefined; - window.reqid = undefined; + lastvolume = null; + lastvolumedata = null; + texture = undefined; } function drawpreview(cfg) { @@ -967,183 +1006,428 @@ function initcanvas() { panel.appendChild(stats.domElement); } - $("#camera-near").on("input", function () { - camera.near = parseFloat($(this).val()); - renderer.render(scene, camera); - controls.update(); - renderer.updateComplete = false; - }); + // $("#camera-near").on("input", function () { + // camera.near = parseFloat($(this).val()); + // renderer.render(scene, camera); + // controls.update(); + // renderer.updateComplete = false; + // }); - $("#camera-far").on("input", function () { - camera.far = parseFloat($(this).val()); - renderer.render(scene, camera); - controls.update(); - renderer.updateComplete = false; - }); + // $("#camera-far").on("input", function () { + // camera.far = parseFloat($(this).val()); + // renderer.render(scene, camera); + // controls.update(); + // renderer.updateComplete = false; + // }); - $("#clim-low").on("input", function () { - $(this).prop( - "title", - "" + - $(this).val() + - " [" + - $(this).prop("min") + - "," + - $(this).prop("max") + - "]" - ); - if (lastvolume !== null) { - let val = lastvolume.material.uniforms["u_clim"].value; - lastvolume.material.uniforms["u_clim"].value.set( - parseFloat($(this).val()), - val.y - ); - renderer.updateComplete = false; - } - }); + // $("#clim-low").on("input", function () { + // $(this).prop( + // "title", + // "" + + // $(this).val() + + // " [" + + // $(this).prop("min") + + // "," + + // $(this).prop("max") + + // "]" + // ); + // if (lastvolume !== null) { + // let val = lastvolume.material.uniforms["u_clim"].value; + // lastvolume.material.uniforms["u_clim"].value.set( + // parseFloat($(this).val()), + // val.y + // ); + // renderer.updateComplete = false; + // } + // }); - $("#clim-hi").on("input", function () { - $(this).prop( - "title", - "" + - $(this).val() + - " [" + - $(this).prop("min") + - "," + - $(this).prop("max") + - "]" - ); - if (lastvolume !== null) { - let val = lastvolume.material.uniforms["u_clim"].value; - lastvolume.material.uniforms["u_clim"].value.set( - val.x, - parseFloat($(this).val()) - ); - renderer.updateComplete = false; - } - }); + // $("#clim-hi").on("input", function () { + // $(this).prop( + // "title", + // "" + + // $(this).val() + + // " [" + + // $(this).prop("min") + + // "," + + // $(this).prop("max") + + // "]" + // ); + // if (lastvolume !== null) { + // let val = lastvolume.material.uniforms["u_clim"].value; + // lastvolume.material.uniforms["u_clim"].value.set( + // val.x, + // parseFloat($(this).val()) + // ); + // renderer.updateComplete = false; + // } + // }); - $("#isothreshold").on("input", function () { - $(this).prop( - "title", - "" + - $(this).val() + - " [" + - $(this).prop("min") + - "," + - $(this).prop("max") + - "]" - ); - if (lastvolume !== null) { - let val = lastvolume.material.uniforms["u_renderthreshold"].value; - lastvolume.material.uniforms["u_renderthreshold"].value = parseFloat( - $(this).val() - ); - renderer.updateComplete = false; - } - }); + // $("#isothreshold").on("input", function () { + // $(this).prop( + // "title", + // "" + + // $(this).val() + + // " [" + + // $(this).prop("min") + + // "," + + // $(this).prop("max") + + // "]" + // ); + // if (lastvolume !== null) { + // let val = lastvolume.material.uniforms["u_renderthreshold"].value; + // lastvolume.material.uniforms["u_renderthreshold"].value = parseFloat( + // $(this).val() + // ); + // renderer.updateComplete = false; + // } + // }); - $("#mip-radio-button").on("change", function () { - if (lastvolume !== null) { - const unfs = lastvolume.material.uniforms; - lastvolume.material = new THREE.ShaderMaterial({ - uniforms: THREE.UniformsUtils.clone(MipRenderShader.uniforms), - vertexShader: MipRenderShader.vertexShader, - fragmentShader: MipRenderShader.fragmentShader, - side: THREE.BackSide, - }); - lastvolume.material.uniforms = unfs; - renderer.updateComplete = false; - } - }); + // $("#mip-radio-button").on("change", function () { + // if (lastvolume !== null) { + // const unfs = lastvolume.material.uniforms; + // lastvolume.material = new THREE.ShaderMaterial({ + // uniforms: THREE.UniformsUtils.clone(MipRenderShader.uniforms), + // vertexShader: MipRenderShader.vertexShader, + // fragmentShader: MipRenderShader.fragmentShader, + // side: THREE.BackSide, + // }); + // lastvolume.material.uniforms = unfs; + // renderer.updateComplete = false; + // } + // }); - $("#iso-radio-button").on("change", function () { - if (lastvolume !== null) { - const unfs = lastvolume.material.uniforms; - lastvolume.material = new THREE.ShaderMaterial({ - uniforms: THREE.UniformsUtils.clone(IsoRenderShader.uniforms), - vertexShader: IsoRenderShader.vertexShader, - fragmentShader: IsoRenderShader.fragmentShader, - side: THREE.BackSide, - }); - lastvolume.material.uniforms = unfs; - renderer.updateComplete = false; - } - }); + // $("#iso-radio-button").on("change", function () { + // if (lastvolume !== null) { + // const unfs = lastvolume.material.uniforms; + // lastvolume.material = new THREE.ShaderMaterial({ + // uniforms: THREE.UniformsUtils.clone(IsoRenderShader.uniforms), + // vertexShader: IsoRenderShader.vertexShader, + // fragmentShader: IsoRenderShader.fragmentShader, + // side: THREE.BackSide, + // }); + // lastvolume.material.uniforms = unfs; + // renderer.updateComplete = false; + // } + // }); - $("#interp-radio-button").on("change", function () { - if (lastvolume !== null) { - const unfs = lastvolume.material.uniforms; - lastvolume.material = new THREE.RawShaderMaterial(InterpRenderShader()); - lastvolume.material.uniforms = unfs; - lastvolume.material.uniforms.cameraPos.value.copy(camera.position); - renderer.updateComplete = false; - } - }); + // $("#interp-radio-button").on("change", function () { + // if (lastvolume !== null) { + // const unfs = lastvolume.material.uniforms; + // lastvolume.material = new THREE.RawShaderMaterial(InterpRenderShader()); + // lastvolume.material.uniforms = unfs; + // lastvolume.material.uniforms.cameraPos.value.copy(camera.position); + // renderer.updateComplete = false; + // } + // }); - $("#cross-x-low").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-x-low").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#cross-y-low").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-y-low").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#cross-z-low").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-z-low").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#cross-x-hi").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-x-hi").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#cross-y-hi").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-y-hi").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#cross-z-hi").on("input", function () { - setcrosssectionsizes(this); - }); + // $("#cross-z-hi").on("input", function () { + // setcrosssectionsizes(this); + // }); - $("#x_thickness, #y_thickness, #z_thickness").on("input", function () { - let eid = $(this).attr("id"); - let linkedeid1 = eid.replace(/_thickness/, "-low").replace(/^/, "cross-"); - let linkedeid2 = eid.replace(/_thickness/, "-hi").replace(/^/, "cross-"); - if ($(this).val() == 0) { - $("#" + linkedeid1).val(0); - $("#" + linkedeid2).val(1); - } else { - $("#" + linkedeid1).val( - ($("#" + linkedeid1).val() + $("#" + linkedeid2).val()) * 0.5 + // $("#x_thickness, #y_thickness, #z_thickness").on("input", function () { + // let eid = $(this).attr("id"); + // let linkedeid1 = eid.replace(/_thickness/, "-low").replace(/^/, "cross-"); + // let linkedeid2 = eid.replace(/_thickness/, "-hi").replace(/^/, "cross-"); + // if ($(this).val() == 0) { + // $("#" + linkedeid1).val(0); + // $("#" + linkedeid2).val(1); + // } else { + // $("#" + linkedeid1).val( + // ($("#" + linkedeid1).val() + $("#" + linkedeid2).val()) * 0.5 + // ); + // } + // setcrosssectionsizes($("#" + linkedeid1)); + // }); + + // $("#pos-x-view").on("click", function () { + // setControlAngles((Math.PI * 90) / 180, (Math.PI * 90) / 180); + // renderer.updateComplete = false; + // }); + + // $("#neg-x-view").on("click", function () { + // setControlAngles((Math.PI * 90) / 180, (Math.PI * 270) / 180); + // }); + + // $("#pos-y-view").on("click", function () { + // setControlAngles((Math.PI * 90) / 180, (Math.PI * 180) / 180); + // }); + + // $("#neg-y-view").on("click", function () { + // setControlAngles((Math.PI * 90) / 180, (Math.PI * 0) / 180); + // }); + + // $("#pos-z-view").on("click", function () { + // setControlAngles(0, 0); + // }); + + // $("#neg-z-view").on("click", function () { + // setControlAngles((Math.PI * 180) / 180, 0); + // }); + + $("#camera-near") + .off("input") + .on("input", function () { + camera.near = parseFloat($(this).val()); + renderer.render(scene, camera); + controls.update(); + renderer.updateComplete = false; + }); + + $("#camera-far") + .off("input") + .on("input", function () { + camera.far = parseFloat($(this).val()); + renderer.render(scene, camera); + controls.update(); + renderer.updateComplete = false; + }); + + $("#clim-low") + .off("input") + .on("input", function () { + $(this).prop( + "title", + "" + + $(this).val() + + " [" + + $(this).prop("min") + + "," + + $(this).prop("max") + + "]" ); - } - setcrosssectionsizes($("#" + linkedeid1)); - }); + if (lastvolume !== null) { + let val = lastvolume.material.uniforms["u_clim"].value; + lastvolume.material.uniforms["u_clim"].value.set( + parseFloat($(this).val()), + val.y + ); + renderer.updateComplete = false; + } + }); - $("#pos-x-view").on("click", function () { - setControlAngles((Math.PI * 90) / 180, (Math.PI * 90) / 180); - renderer.updateComplete = false; - }); + $("#clim-hi") + .off("input") + .on("input", function () { + $(this).prop( + "title", + "" + + $(this).val() + + " [" + + $(this).prop("min") + + "," + + $(this).prop("max") + + "]" + ); + if (lastvolume !== null) { + let val = lastvolume.material.uniforms["u_clim"].value; + lastvolume.material.uniforms["u_clim"].value.set( + val.x, + parseFloat($(this).val()) + ); + renderer.updateComplete = false; + } + }); - $("#neg-x-view").on("click", function () { - setControlAngles((Math.PI * 90) / 180, (Math.PI * 270) / 180); - }); + $("#isothreshold") + .off("input") + .on("input", function () { + $(this).prop( + "title", + "" + + $(this).val() + + " [" + + $(this).prop("min") + + "," + + $(this).prop("max") + + "]" + ); + if (lastvolume !== null) { + lastvolume.material.uniforms["u_renderthreshold"].value = parseFloat( + $(this).val() + ); + renderer.updateComplete = false; + } + }); - $("#pos-y-view").on("click", function () { - setControlAngles((Math.PI * 90) / 180, (Math.PI * 180) / 180); - }); + $("#mip-radio-button") + .off("change") + .on("change", function () { + if (lastvolume !== null) { + const unfs = lastvolume.material.uniforms; + lastvolume.material = new THREE.ShaderMaterial({ + uniforms: THREE.UniformsUtils.clone(MipRenderShader.uniforms), + vertexShader: MipRenderShader.vertexShader, + fragmentShader: MipRenderShader.fragmentShader, + side: THREE.BackSide, + }); + lastvolume.material.uniforms = unfs; + renderer.updateComplete = false; + } + }); - $("#neg-y-view").on("click", function () { - setControlAngles((Math.PI * 90) / 180, (Math.PI * 0) / 180); - }); + $("#iso-radio-button") + .off("change") + .on("change", function () { + if (lastvolume !== null) { + const unfs = lastvolume.material.uniforms; + lastvolume.material = new THREE.ShaderMaterial({ + uniforms: THREE.UniformsUtils.clone(IsoRenderShader.uniforms), + vertexShader: IsoRenderShader.vertexShader, + fragmentShader: IsoRenderShader.fragmentShader, + side: THREE.BackSide, + }); + lastvolume.material.uniforms = unfs; + renderer.updateComplete = false; + } + }); - $("#pos-z-view").on("click", function () { - setControlAngles(0, 0); - }); + $("#interp-radio-button") + .off("change") + .on("change", function () { + if (lastvolume !== null) { + const unfs = lastvolume.material.uniforms; + lastvolume.material = new THREE.RawShaderMaterial(InterpRenderShader()); + lastvolume.material.uniforms = unfs; + lastvolume.material.uniforms.cameraPos.value.copy(camera.position); + renderer.updateComplete = false; + } + }); - $("#neg-z-view").on("click", function () { - setControlAngles((Math.PI * 180) / 180, 0); - }); + $("#cross-x-low") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + $("#cross-y-low") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + $("#cross-z-low") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + $("#cross-x-hi") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + $("#cross-y-hi") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + $("#cross-z-hi") + .off("input") + .on("input", function () { + setcrosssectionsizes(this); + }); + + $("#x_thickness, #y_thickness, #z_thickness") + .off("input") + .on("input", function () { + let eid = $(this).attr("id"); + let linkedeid1 = eid.replace(/_thickness/, "-low").replace(/^/, "cross-"); + let linkedeid2 = eid.replace(/_thickness/, "-hi").replace(/^/, "cross-"); + if ($(this).val() == 0) { + $("#" + linkedeid1).val(0); + $("#" + linkedeid2).val(1); + } else { + $("#" + linkedeid1).val( + ($("#" + linkedeid1).val() + $("#" + linkedeid2).val()) * 0.5 + ); + } + setcrosssectionsizes($("#" + linkedeid1)); + }); + + $("#pos-x-view") + .off("click") + .on("click", function () { + setControlAngles((Math.PI * 90) / 180, (Math.PI * 90) / 180); + renderer.updateComplete = false; + }); + $("#neg-x-view") + .off("click") + .on("click", function () { + setControlAngles((Math.PI * 90) / 180, (Math.PI * 270) / 180); + }); + $("#pos-y-view") + .off("click") + .on("click", function () { + setControlAngles((Math.PI * 90) / 180, (Math.PI * 180) / 180); + }); + $("#neg-y-view") + .off("click") + .on("click", function () { + setControlAngles((Math.PI * 90) / 180, (Math.PI * 0) / 180); + }); + $("#pos-z-view") + .off("click") + .on("click", function () { + setControlAngles(0, 0); + }); + $("#neg-z-view") + .off("click") + .on("click", function () { + setControlAngles((Math.PI * 180) / 180, 0); + }); + + $("#cross-t") + .off("mouseup") + .on("mouseup", function () { + $(this).prop( + "title", + "" + + $(this).val() + + " [" + + $(this).prop("min") + + "," + + $(this).prop("max") + + "]" + ); + if (lastvolume !== null && lastvolumedata !== undefined) { + let dim = lastvolumedim; + let offset = + Math.min($(this).val(), dim[3] - 2) * dim[0] * dim[1] * dim[2]; + let texture = new THREE.Data3DTexture( + lastvolumedata.selection.data.slice( + offset - 1, + offset + dim[0] * dim[1] * dim[2] - 1 + ), + dim[0], + dim[1], + dim[2] + ); + texture.format = THREE.RedFormat; + texture.type = texture_dtype[lastvolumedata.dtype]; + texture.minFilter = texture.magFilter = THREE.LinearFilter; + texture.unpackAlignment = 1; + texture.needsUpdate = true; + lastvolume.material.uniforms["u_data"].value = texture; + renderer.updateComplete = false; + } + }); $("#cross-t").on("mouseup", function () { $(this).prop( From e54ea6f058496d17201552a454f8c83d3e9458fe Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 2 Apr 2026 09:32:15 -0400 Subject: [PATCH 03/61] fix: resolve memory leaks in 2D plot viewer --- src/utils/preview.js | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/utils/preview.js b/src/utils/preview.js index 496e855..02f3908 100644 --- a/src/utils/preview.js +++ b/src/utils/preview.js @@ -33,6 +33,7 @@ var lastvolume = null; var lastvolumedata = null; var lastvolumedim = []; var lastclim = 0; +var uplotInstance = null; var reqid = undefined; var canvas = null; @@ -133,6 +134,12 @@ function destroyPreview() { lastvolume = null; lastvolumedata = null; texture = undefined; + + if (uplotInstance !== null) { + uplotInstance.destroy(); + uplotInstance = null; + } + $("#chartpanel").hide().html(""); } function drawpreview(cfg) { @@ -428,22 +435,32 @@ function dopreview(key, idx, isinternal, hastime) { ? "y" + i : hastime[i]; } - let u = new uPlot(opts, plotdata, document.getElementById("plotchart")); + // let u = new uPlot(opts, plotdata, document.getElementById("plotchart")); + if (uplotInstance !== null) { + uplotInstance.destroy(); + uplotInstance = null; + } + uplotInstance = new uPlot( + opts, + plotdata, + document.getElementById("plotchart") + ); } else { - let u = new uPlot( + // let u = new uPlot( + // opts, + // [[...Array(dataroot.length).keys()], dataroot], + // document.getElementById("plotchart") + // ); + if (uplotInstance !== null) { + uplotInstance.destroy(); + uplotInstance = null; + } + uplotInstance = new uPlot( opts, [[...Array(dataroot.length).keys()], dataroot], document.getElementById("plotchart") ); } - // add spinner - // --- NEW LOGIC for 2D plot --- - // Signal that the 2D plot has just been created and is now visible. - // if (typeof window.__onPreviewReady === "function") { - // window.__onPreviewReady(); - // window.__onPreviewReady = null; // Clean up to prevent accidental re-firing - // } - // --- END NEW LOGIC --- // for spinner // --- Signal React that 2D preview is ready --- From ecf15ebf7146590eb57dcf4d1988290b8aab6bf5 Mon Sep 17 00:00:00 2001 From: Qianqian Fang Date: Thu, 2 Apr 2026 11:44:01 -0400 Subject: [PATCH 04/61] [debug] print github action server IP --- .github/workflows/build-deploy-zodiac.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-deploy-zodiac.yml b/.github/workflows/build-deploy-zodiac.yml index 57a049d..d6d86e6 100644 --- a/.github/workflows/build-deploy-zodiac.yml +++ b/.github/workflows/build-deploy-zodiac.yml @@ -18,6 +18,9 @@ jobs: - name: Check out the repository uses: actions/checkout@v3 + - name: Check IP + run: curl https://api.ipify.org + - name: Install dependencies run: yarn install --frozen-lockfile From 66f6e435ead30e55958fba297e4255b34a2283e9 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 2 Apr 2026 13:51:51 -0400 Subject: [PATCH 05/61] feat: add executorHelpers and plannerHelpers --- .../DatasetOrganizer/utils/executorHelpers.ts | 522 ++++++++++++++++++ .../DatasetOrganizer/utils/fileAnalyzers.ts | 332 ++--------- .../utils/filenameTokenizer.ts | 505 +++++++++++++++-- .../DatasetOrganizer/utils/plannerHelpers.ts | 0 4 files changed, 1030 insertions(+), 329 deletions(-) create mode 100644 src/components/User/Dashboard/DatasetOrganizer/utils/executorHelpers.ts create mode 100644 src/components/User/Dashboard/DatasetOrganizer/utils/plannerHelpers.ts diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/executorHelpers.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/executorHelpers.ts new file mode 100644 index 0000000..782dee5 --- /dev/null +++ b/src/components/User/Dashboard/DatasetOrganizer/utils/executorHelpers.ts @@ -0,0 +1,522 @@ +// src/components/DatasetOrganizer/utils/executorHelpers.ts +// +// Portable helper functions from autobidsify/converters/executor.py +// +// What is NOT here (intentionally — requires server-side CLI): +// execute_bids_plan() — file copy/conversion operations +// convert_mat_to_snirf() — binary .mat read + .snirf write +// run_dcm2niix_batch() — dcm2niix subprocess +// convert_jnifti_to_nifti() — nibabel NIfTI write +// +// What IS here (useful client-side for plan validation + preview): +// sanitizeBidsLabel() mirrors _sanitize_bids_label() +// normalizeFilename() mirrors _normalize_filename() +// extractAcqLabel() mirrors _extract_acq_label() +// selectPreferredFile() mirrors _select_preferred_file() +// matchGlobPattern() mirrors _match_glob_pattern() +// inferScanType() mirrors infer_scan_type_from_filepath() +// inferSubdirectory() mirrors infer_subdirectory_from_suffix() +// categorizeScanType() mirrors categorize_scan_type() +// analyzeFilepathUniversal() mirrors analyze_filepath_universal() +// validatePlanCoverage() NEW — uses matchGlobPattern to check LLM patterns + +// ============================================================================ +// sanitizeBidsLabel() +// Mirrors _sanitize_bids_label() in executor.py +// Removes all non-alphanumeric characters from a BIDS entity value +// e.g. "mental_arithmetic" → "mentalarithmetic" +// ============================================================================ + +export const sanitizeBidsLabel = (label: string): string => + label.replace(/[^a-zA-Z0-9]/g, ""); + +// ============================================================================ +// normalizeFilename() +// Mirrors _normalize_filename() in executor.py +// +// Strips extensions and trailing sequence numbers. +// Used to identify DICOM series and detect format duplicates. +// +// Examples: +// "VHFCT1mm-Hip (134).dcm" → "vhfct1mm-hip" +// "scan_mprage_anonymized.nii.gz" → "scan_mprage_anonymized" +// "scan_001.dcm" → "scan" +// ============================================================================ + +export const normalizeFilename = (filepath: string): string => { + let name = filepath.split("/").pop()!; + + // Strip all extensions (up to 6 chars) + while (name.includes(".") && name.split(".").pop()!.length <= 6) { + name = name.substring(0, name.lastIndexOf(".")); + } + + // Strip trailing " (N)" + name = name.replace(/\s*\(\d+\)\s*$/, ""); + // Strip trailing _NNN or -NNN + name = name.replace(/[_\-]\d+$/, ""); + + return name.trim().toLowerCase(); +}; + +// ============================================================================ +// extractAcqLabel() +// Mirrors _extract_acq_label() in executor.py +// +// Derives a short, clean acq- label from a normalized DICOM filename. +// Keeps the last meaningful alphabetic token (body part or scan descriptor). +// +// Examples: +// "vhfct1mmankle" → "ankle" +// "vhfct1mmhead" → "head" +// "vhmct1mmhip" → "hip" +// "scanmprage" → "mprage" +// ============================================================================ + +export const extractAcqLabel = (normalizedFname: string): string => { + const skip = new Set(["vhf", "vhm", "ct", "mr", "mri", "mm", "scan", "the"]); + const tokens = normalizedFname.match(/[a-z]+/g) || []; + const meaningful = tokens.filter((t) => t.length > 2 && !skip.has(t)); + + if (meaningful.length > 0) { + return meaningful[meaningful.length - 1]; // last = body part + } + return normalizedFname.slice(0, 20); // fallback: cap at 20 chars +}; + +// ============================================================================ +// selectPreferredFile() +// Mirrors _select_preferred_file() in executor.py +// +// Priority: NIfTI dir > non-BRIK > shortest path > alphabetical +// ============================================================================ + +export const selectPreferredFile = (files: string[]): string | null => { + if (files.length === 0) return null; + if (files.length === 1) return files[0]; + + const priority = (f: string): [number, number, number, string] => { + const parts = f.toLowerCase().split("/"); + return [ + parts.some((p) => p.includes("nifti")) ? 0 : 1, + parts.some((p) => p.includes("brik")) ? 1 : 0, + parts.length, + f, + ]; + }; + + return [...files].sort((a, b) => { + const [a0, a1, a2, a3] = priority(a); + const [b0, b1, b2, b3] = priority(b); + if (a0 !== b0) return a0 - b0; + if (a1 !== b1) return a1 - b1; + if (a2 !== b2) return a2 - b2; + return a3.localeCompare(b3); + })[0]; +}; + +// ============================================================================ +// matchGlobPattern() +// Mirrors _match_glob_pattern() in executor.py +// +// Supported patterns: +// "**/*.nii.gz" → any .nii.gz at any depth +// "**/BRIK/**" → any file inside a BRIK directory +// "*token*" → filepath contains token +// "*.ext" → filename ends with extension +// "token*" → filename starts with token +// "plain" → substring anywhere in path (fallback) +// ============================================================================ + +export const matchGlobPattern = ( + filepath: string, + pattern: string +): boolean => { + const fp = filepath.toLowerCase(); + const pat = pattern.toLowerCase(); + const parts = fp.split("/"); + const filename = parts[parts.length - 1]; + + // **/TOKEN/** — directory component match + if (pat.startsWith("**/") && pat.endsWith("/**")) { + const token = pat.slice(3, -3); + return parts.slice(0, -1).includes(token); + } + + // **/*.ext — any depth extension match + if (pat.startsWith("**/")) { + const suffix = pat.slice(3); + if (suffix.startsWith("*.")) return fp.endsWith(suffix.slice(1)); + return fp.includes(suffix); + } + + // *token* — substring in full path + if (pat.startsWith("*") && pat.endsWith("*")) { + return fp.includes(pat.slice(1, -1)); + } + + // *.ext — extension match on filename only + if (pat.startsWith("*.")) { + return filename.endsWith(pat.slice(1)); + } + + // token* — filename prefix + if (pat.endsWith("*")) { + return filename.startsWith(pat.slice(0, -1)); + } + + // fallback — substring anywhere in path + return fp.includes(pat); +}; + +// ============================================================================ +// inferScanType() +// Mirrors infer_scan_type_from_filepath() in executor.py +// +// Priority: +// 1. LLM filename_rules from BIDSPlan +// 2. BIDS entities already in filename (ses-, task-, acq-, run-) +// 3. Keyword detection in path +// 4. Extension fallback +// ============================================================================ + +interface ScanTypeResult { + suffix: string; + subdirectory: string; + category: string; +} + +export const inferScanType = ( + filepath: string, + filenameRules: any[] = [] +): ScanTypeResult => { + const pathLower = filepath.toLowerCase(); + const filename = filepath.split("/").pop()!; + const fnameLow = filename.toLowerCase(); + + // ── Priority 1: LLM filename_rules ────────────────────────────────── + for (const rule of filenameRules) { + try { + const mp = (rule.match_pattern || "").replace(/\\\\/g, "\\"); + if (!new RegExp(mp, "i").test(filename)) continue; + + const template: string = rule.bids_template || ""; + const m = template.match(/sub-[^_]+_(.*?)\.(nii\.gz|snirf|nii)/); + if (!m) continue; + + let raw = m[1]; + // Remove placeholder entities + raw = raw + .replace(/ses-X_?/g, "") + .replace(/task-X_?/g, "") + .replace(/^_|_$/g, ""); + + // Remove spurious ses- if no ses- dir in path + if ( + /ses-[A-Za-z0-9]+/.test(raw) && + !/\/ses-[A-Za-z0-9]+\//.test(filepath) + ) { + raw = raw.replace(/ses-[A-Za-z0-9]+_?/g, "").replace(/^_|_$/g, ""); + } + + if (raw) { + // Sanitize entity values — mirrors _sanitize_suffix() in executor.py + // "task-mental_arithmetic_nirs" → "task-mentalarithmetic_nirs" + raw = raw.replace( + /([a-zA-Z]+-)(.+?)(?=_[a-zA-Z]+-|_[a-zA-Z]+$|$)/g, + (_match, key, val) => key + sanitizeBidsLabel(val) + ); + const subdir = inferSubdirectory(raw); + return { + suffix: raw, + subdirectory: subdir, + category: categorizeScanType(raw), + }; + } + } catch { + continue; + } + } + + // ── Priority 2: BIDS entities already in filename ──────────────────── + const entities: Record = {}; + for (const [key, pattern] of [ + ["ses", /ses-([A-Za-z0-9]+)/], + ["task", /task-([A-Za-z0-9]+)/], + ["acq", /acq-([A-Za-z0-9]+)/], + ["run", /run-([A-Za-z0-9]+)/], + ] as [string, RegExp][]) { + const match = filename.match(pattern); + if (match) entities[key] = match[1]; + } + + // Infer task from filename keywords when no task- entity present + if (!entities.task) { + const nameNoExt = fnameLow.replace(/\.[^.]+$/, ""); + if (/rest|resting/.test(nameNoExt)) entities.task = "rest"; + else if (/finger|tapping|fingertap/.test(nameNoExt)) + entities.task = "fingertapping"; + else if (/walking|walk/.test(nameNoExt)) entities.task = "walking"; + else if (/motor|tap/.test(nameNoExt)) entities.task = "motor"; + } + + let modalityLabel: string | null = null; + let subdir = "anat"; + + if (fnameLow.endsWith(".snirf") || fnameLow.includes("nirs")) { + modalityLabel = "nirs"; + subdir = "nirs"; + } else if (/t1w|t1/.test(fnameLow)) { + modalityLabel = "T1w"; + subdir = "anat"; + } else if (/t2w|t2/.test(fnameLow)) { + modalityLabel = "T2w"; + subdir = "anat"; + } else if (/bold|func/.test(fnameLow)) { + modalityLabel = "bold"; + subdir = "func"; + } else if (/dwi/.test(fnameLow)) { + modalityLabel = "dwi"; + subdir = "dwi"; + } + + // BIDS rule: task-* scans go in func/ (unless nirs) + if (subdir !== "nirs" && (entities.task || pathLower.includes("func/"))) { + subdir = "func"; + if (!modalityLabel) modalityLabel = "bold"; + } + + if (Object.keys(entities).length > 0 || modalityLabel) { + const parts: string[] = []; + for (const key of ["ses", "task", "acq", "run"]) { + if (entities[key]) + parts.push(`${key}-${sanitizeBidsLabel(entities[key])}`); + } + if (modalityLabel) parts.push(modalityLabel); + if (parts.length > 0) { + const suffix = parts.join("_"); + return { + suffix, + subdirectory: subdir, + category: categorizeScanType(suffix), + }; + } + } + + // ── Priority 3: Heuristic path keywords ───────────────────────────── + if (/anat|mprage|t1w/.test(pathLower)) + return { suffix: "T1w", subdirectory: "anat", category: "anatomical" }; + if (/func|bold/.test(pathLower)) { + const m = pathLower.match(/task[_-]([a-z0-9]+)/); + const suffix = m ? `task-${m[1]}_bold` : "task-rest_bold"; + return { suffix, subdirectory: "func", category: "functional" }; + } + if (pathLower.includes("rest")) + return { + suffix: "task-rest_bold", + subdirectory: "func", + category: "functional", + }; + if (/nirs|fnirs|\.snirf/.test(pathLower)) + return { suffix: "nirs", subdirectory: "nirs", category: "functional" }; + if (pathLower.includes("dwi")) + return { suffix: "dwi", subdirectory: "dwi", category: "diffusion" }; + + // ── Priority 4: Extension fallback ────────────────────────────────── + if (fnameLow.endsWith(".snirf")) + return { suffix: "nirs", subdirectory: "nirs", category: "functional" }; + if (fnameLow.endsWith(".nii") || fnameLow.endsWith(".nii.gz")) + return { suffix: "T1w", subdirectory: "anat", category: "anatomical" }; + + return { suffix: "unknown", subdirectory: "anat", category: "unknown" }; +}; + +// ============================================================================ +// inferSubdirectory() +// Mirrors infer_subdirectory_from_suffix() in executor.py +// ============================================================================ + +export const inferSubdirectory = (suffix: string): string => { + const s = suffix.toLowerCase(); + if (s.includes("t1w") || s.includes("t2w")) return "anat"; + if (s.includes("bold")) return "func"; + if (s.includes("nirs")) return "nirs"; + if (s.includes("dwi")) return "dwi"; + return "anat"; +}; + +// ============================================================================ +// categorizeScanType() +// Mirrors categorize_scan_type() in executor.py +// ============================================================================ + +export const categorizeScanType = (suffix: string): string => { + const s = suffix.toLowerCase(); + if (s.includes("t1w") || s.includes("t2w")) return "anatomical"; + if (s.includes("bold") || s.includes("nirs")) return "functional"; + if (s.includes("dwi")) return "diffusion"; + return "unknown"; +}; + +// ============================================================================ +// analyzeFilepathUniversal() +// Mirrors analyze_filepath_universal() in executor.py +// +// Determines BIDS subject ID and output filename for one source file. +// Used for plan preview — shows user what each file will become. +// ============================================================================ + +export interface FilepathAnalysis { + subject_id: string; + scan_type_suffix: string; + bids_filename: string; + subdirectory: string; + scan_category: string; + original_filepath: string; + modality: string; +} + +export const analyzeFilepathUniversal = ( + filepath: string, + assignmentRules: any[], + filenameRules: any[], + modality: string = "mri" +): FilepathAnalysis => { + const filename = filepath.split("/").pop()!; + const pathParts = filepath.split("/"); + let subjectId: string | null = null; + + // Priority 1: match glob patterns + for (const rule of assignmentRules) { + for (const pat of rule.match || []) { + if (matchGlobPattern(filepath, pat)) { + subjectId = rule.subject; + break; + } + } + if (subjectId) break; + } + + // Priority 2: original substring match + if (!subjectId) { + for (const rule of assignmentRules) { + const orig: string = rule.original || ""; + if (orig && filepath.toLowerCase().includes(orig.toLowerCase())) { + subjectId = rule.subject; + break; + } + } + } + + // Priority 3: prefix match + if (!subjectId) { + for (const rule of assignmentRules) { + const pfx: string = rule.prefix || ""; + if (pfx && filename.toLowerCase().startsWith(pfx.toLowerCase())) { + subjectId = rule.subject; + break; + } + } + } + + // Priority 4: sub-XX already in path + if (!subjectId) { + for (const part of pathParts) { + const m = part.match(/sub[_-]?(\w+)/i); + if (m) { + subjectId = m[1]; + break; + } + } + } + + // Fallback + if (!subjectId) subjectId = "unknown"; + + // Strip accidental sub- prefix + if (subjectId.startsWith("sub-")) subjectId = subjectId.slice(4); + + const scanInfo = inferScanType(filepath, filenameRules); + const ext = modality === "nirs" ? ".snirf" : ".nii.gz"; + const bidsFilename = `sub-${subjectId}_${scanInfo.suffix}${ext}`; + + return { + subject_id: subjectId, + scan_type_suffix: scanInfo.suffix, + bids_filename: bidsFilename, + subdirectory: scanInfo.subdirectory, + scan_category: scanInfo.category, + original_filepath: filepath, + modality, + }; +}; + +// ============================================================================ +// validatePlanCoverage() +// NEW — not in Python (Python validates at runtime, we validate at plan-time) +// +// Checks that the LLM's match patterns in BIDSPlan actually cover the +// sample files from the evidence bundle. Warns about uncovered files. +// +// Used in plannerHelpers.ts after buildBidsPlan() to surface issues +// before the user downloads the ZIP. +// ============================================================================ + +export interface PlanCoverageResult { + covered: string[]; + uncovered: string[]; + coveragePercent: number; + warnings: string[]; +} + +export const validatePlanCoverage = ( + sampleFiles: string[], + mappings: any[] +): PlanCoverageResult => { + const covered: string[] = []; + const uncovered: string[] = []; + const warnings: string[] = []; + + for (const filepath of sampleFiles) { + let isCovered = false; + + for (const mapping of mappings) { + const patterns: string[] = mapping.match || []; + const excludes: string[] = mapping.exclude || []; + + const isExcluded = excludes.some((ex) => matchGlobPattern(filepath, ex)); + if (isExcluded) continue; + + const isMatched = patterns.some((pat) => matchGlobPattern(filepath, pat)); + if (isMatched) { + isCovered = true; + break; + } + } + + if (isCovered) covered.push(filepath); + else uncovered.push(filepath); + } + + if (uncovered.length > 0) { + warnings.push( + `${uncovered.length} sample file(s) not covered by any mapping pattern.` + ); + for (const f of uncovered.slice(0, 5)) { + warnings.push(` Uncovered: ${f}`); + } + if (uncovered.length > 5) { + warnings.push(` ... and ${uncovered.length - 5} more`); + } + } + + return { + covered, + uncovered, + coveragePercent: + sampleFiles.length > 0 + ? Math.round((covered.length / sampleFiles.length) * 100) + : 100, + warnings, + }; +}; diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/fileAnalyzers.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/fileAnalyzers.ts index 76142c0..c3c89b0 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/utils/fileAnalyzers.ts +++ b/src/components/User/Dashboard/DatasetOrganizer/utils/fileAnalyzers.ts @@ -1,54 +1,60 @@ // src/components/DatasetOrganizer/utils/fileAnalyzers.ts +// VFS adapter layer because NeuroJSON.io needs to work with FileItem[] objects +// No single Python mirror. Functions map to: +// categorizeFile() → executor.py (infer_subdirectory_from_suffix, categorize_scan_type) +// detectModality() → evidence.py (detect_kind) + constants.py (MODALITY_*) +// getCountsByExtension() → evidence.py (by_ext dict construction) +// getUserContextText() → evidence.py (_extract_document_content + documents[] assembly) import { FileItem } from "redux/projects/types/projects.interface"; -/** - * Categorize a file based on its name and type - * Returns detailed scan category (anatomical-T1w, functional-bold, etc.) - */ +// ============================================================================ +// categorizeFile() +// UI display function — determines file label/color in FileTree. +// +// NOT the same as inferScanType() in executorHelpers.ts: +// categorizeFile() → "what to show in the UI" +// inferScanType() → "what BIDS filename to generate" +// +// Partial mirror of executor.py → infer_subdirectory_from_suffix() +// + categorize_scan_type() +// ============================================================================ export const categorizeFile = (file: FileItem): string => { const name = file.name.toLowerCase(); // Functional scans (task-based) - if (name.includes("task-") && name.includes("bold")) { - return "functional-bold"; - } - if (name.endsWith(".snirf")) { - return "functional-nirs"; - } - + if (name.includes("task-") && name.includes("bold")) return "functional-bold"; + if (name.endsWith(".snirf")) return "functional-nirs"; if (name.endsWith(".nirs")) return "functional-nirs"; if (name.endsWith(".mat")) return "functional-nirs"; // Anatomical scans - if (name.includes("t1w")) { - return "anatomical-T1w"; - } - if (name.includes("t2w") || name.includes("inplanet2")) { + if (name.includes("t1w")) return "anatomical-T1w"; + if (name.includes("t2w") || name.includes("inplanet2")) return "anatomical-T2w"; - } - if (name.includes("flair")) { - return "anatomical-FLAIR"; - } - + if (name.includes("flair")) return "anatomical-FLAIR"; if (name.endsWith(".dcm")) return "anatomical-dicom"; + // JNIfTI — mirrors JNIFTI_EXT in constants.py: {'.jnii', '.bnii'} + if (name.endsWith(".jnii") || name.endsWith(".bnii")) + return "anatomical-jnifti"; + // Diffusion - if (name.includes("dwi") || name.includes("diffusion")) { - return "diffusion"; - } + if (name.includes("dwi") || name.includes("diffusion")) return "diffusion"; // Field maps - if (name.includes("fieldmap") || name.includes("fmap")) { - return "fieldmap"; - } + if (name.includes("fieldmap") || name.includes("fmap")) return "fieldmap"; - // Fall back to file type + // Array/HDF5 (non-SNIRF) + if (name.endsWith(".h5") || name.endsWith(".hdf5")) return "array"; + + // Fall back to fileType from fileProcessors.ts return file.fileType || "unknown"; }; -/** - * Detect modality from file collection - */ +// ============================================================================ +// Detect modality from file collection +// Rough equivalent of evidence.py → detect_kind() + constants.py MODALITY_* +// ============================================================================ export const detectModality = (files: FileItem[]): string => { const counts: Record = {}; files.forEach((f) => { @@ -57,31 +63,29 @@ export const detectModality = (files: FileItem[]): string => { }); if (counts.nifti > 0 || counts.dicom > 0) return "mri"; + // FIX: fileProcessors.ts returns "nirs" for .nirs files, not "homer3" if ( counts.hdf5 > 0 || counts.matlab > 0 || - counts.homer3 > 0 || + counts.nirs > 0 || files.some((f) => f.name.endsWith(".snirf")) ) return "nirs"; return "mixed"; }; -/** - * Get file extension counts - */ +// ============================================================================ +// Get file extension counts +// Mirrors evidence.py → by_ext dict construction. +// Uses ".nii.gz" as a single key — mirrors Python: p.name.lower().endswith(".nii.gz") +// ============================================================================ export const getCountsByExtension = ( files: FileItem[] ): Record => { const counts: Record = {}; - // files.forEach((f) => { - // const ext = f.fileType || "unknown"; - // counts[ext] = (counts[ext] || 0) + 1; - // }); files .filter((f) => f.source === "user" && f.type === "file") .forEach((f) => { - // Mirror Python: use ".nii.gz" as a single key for .nii.gz files const name = f.name.toLowerCase(); const ext = name.endsWith(".nii.gz") ? ".nii.gz" @@ -91,9 +95,13 @@ export const getCountsByExtension = ( return counts; }; -/** - * Extract user context from metadata files - */ +// ============================================================================ +// Extract user context from metadata files +// Partial mirror of evidence.py → _extract_document_content() + +// the documents[] assembly in _build_evidence_bundle_internal(). +// +// Python reads files from disk; this reads from VFS FileItem.content. +// ============================================================================ export const getUserContextText = (files: FileItem[]): string => { const readme = files.find((f) => f.name.toLowerCase().includes("readme")); const instructions = files.find( @@ -130,243 +138,3 @@ export const getUserContextText = (files: FileItem[]): string => { }); return parts.join("\n\n"); }; - -/** (not using yet) - * Analyze filename patterns to detect subjects - * (Simplified version inspired by auto-bidsify's filename_tokenizer) - */ -export const analyzeFilenamePatterns = ( - files: FileItem[] -): { - subjectCount: number; - subjectIds: string[]; - hasRunNumbers: boolean; - hasTaskNames: boolean; -} => { - const dataFiles = files.filter((f) => f.type === "file" && !f.isUserMeta); - const subjectIds = new Set(); - let hasRunNumbers = false; - let hasTaskNames = false; - - dataFiles.forEach((f) => { - const name = f.name; - - // Extract subject ID (sub-01, sub-02, etc.) - const subMatch = name.match(/sub-(\d+)/i); - if (subMatch) { - subjectIds.add(subMatch[1]); - } - - // Check for run numbers - if (name.includes("_run-")) { - hasRunNumbers = true; - } - - // Check for task names - if (name.includes("task-")) { - hasTaskNames = true; - } - }); - - return { - subjectCount: subjectIds.size, - subjectIds: Array.from(subjectIds).sort(), - hasRunNumbers, - hasTaskNames, - }; -}; - -// add to fileAnalyzers.ts - -// export interface SubjectRecord { -// original_id: string; -// numeric_id: string; -// site: string | null; -// pattern_name: string; -// file_count: number; -// } - -// export interface SubjectAnalysis { -// success: boolean; -// method: string; -// subject_records: SubjectRecord[]; -// subject_count: number; -// has_site_info: boolean; -// variants_by_subject: Record; -// python_generated_filename_rules: any[]; -// id_mapping: { -// id_mapping: Record; -// reverse_mapping: Record; -// strategy_used: string; -// metadata_columns: string[]; -// }; -// } - -// // mirrors _extract_subjects_from_directory_structure -// const extractFromDirectoryStructure = ( -// allFiles: string[] -// ): Omit | null => { -// const patterns: Array<[RegExp, boolean, number, number | null, string]> = [ -// [/^([A-Za-z]+)_sub(\d+)$/i, true, 2, 1, "site_prefixed"], -// [/^sub-(\d+)$/i, false, 1, null, "standard_bids"], -// [/^subject[_-]?(\d+)$/i, false, 1, null, "simple"], -// [/^(\d{3,})$/, false, 1, null, "numeric_only"], -// ]; - -// const subjectRecords: SubjectRecord[] = []; -// const seenIds = new Set(); - -// for (const filepath of allFiles) { -// const parts = filepath.split("/"); -// for (const part of parts.slice(0, 2)) { -// for (const [ -// regex, -// hasSite, -// idGroup, -// siteGroup, -// patternName, -// ] of patterns) { -// const match = part.match(regex); -// if (match) { -// const originalId = match[0]; -// if (seenIds.has(originalId)) break; -// seenIds.add(originalId); -// subjectRecords.push({ -// original_id: originalId, -// numeric_id: match[idGroup], -// site: hasSite && siteGroup ? match[siteGroup] : null, -// pattern_name: patternName, -// file_count: 0, -// }); -// break; -// } -// } -// } -// } - -// if (subjectRecords.length === 0) return null; - -// subjectRecords.sort((a, b) => { -// const na = parseInt(a.numeric_id) || 0; -// const nb = parseInt(b.numeric_id) || 0; -// return na - nb; -// }); - -// return { -// success: true, -// method: "directory_structure", -// subject_records: subjectRecords, -// subject_count: subjectRecords.length, -// has_site_info: subjectRecords.some((r) => r.site !== null), -// variants_by_subject: {}, -// python_generated_filename_rules: [], -// }; -// }; - -// // mirrors _extract_subjects_from_flat_filenames -// const extractFromFlatFilenames = ( -// allFiles: string[] -// ): Omit | null => { -// const identifierToFiles: Record = {}; - -// for (const filepath of allFiles) { -// const filename = filepath.split("/").pop() || ""; -// const nameNoExt = filename -// .replace(/\.[^/.]+$/, "") -// .replace(/\.nii\.gz$/, ""); -// const match = nameNoExt.match(/^([A-Za-z0-9\-]+)/); -// if (match) { -// const identifier = match[1]; -// if (!identifierToFiles[identifier]) identifierToFiles[identifier] = []; -// identifierToFiles[identifier].push(filepath); -// } -// } - -// if (Object.keys(identifierToFiles).length === 0) return null; - -// const extractNumeric = (id: string): number => { -// const nums = id.match(/\d+/g); -// return nums ? parseInt(nums[nums.length - 1]) : 999999; -// }; - -// const sortedIdentifiers = Object.keys(identifierToFiles).sort( -// (a, b) => extractNumeric(a) - extractNumeric(b) -// ); - -// const subjectRecords: SubjectRecord[] = sortedIdentifiers.map((id, i) => ({ -// original_id: id, -// numeric_id: String(i + 1), -// site: null, -// pattern_name: "dominant_prefix", -// file_count: identifierToFiles[id].length, -// })); - -// return { -// success: true, -// method: "dominant_prefix_fallback", -// subject_records: subjectRecords, -// subject_count: subjectRecords.length, -// has_site_info: false, -// variants_by_subject: {}, -// python_generated_filename_rules: [], -// }; -// }; - -// // mirrors _generate_subject_id_mapping -// const generateIdMapping = ( -// subjectInfo: Omit -// ): SubjectAnalysis["id_mapping"] => { -// const records = subjectInfo.subject_records; -// const idMapping: Record = {}; -// const reverseMapping: Record = {}; - -// // detect already-BIDS format (sub-01, sub-02...) -// const allAlreadyBids = records.every((r) => /^sub-\w+$/i.test(r.original_id)); - -// if (allAlreadyBids) { -// for (const rec of records) { -// const bidsId = rec.original_id.replace(/^sub-/i, ""); -// idMapping[rec.original_id] = bidsId; -// reverseMapping[bidsId] = rec.original_id; -// } -// return { -// id_mapping: idMapping, -// reverse_mapping: reverseMapping, -// strategy_used: "already_bids", -// metadata_columns: [], -// }; -// } - -// // numeric strategy -// for (let i = 0; i < records.length; i++) { -// const orig = records[i].original_id; -// const bidsId = String(i + 1); -// idMapping[orig] = bidsId; -// reverseMapping[bidsId] = orig; -// } - -// return { -// id_mapping: idMapping, -// reverse_mapping: reverseMapping, -// strategy_used: "numeric", -// metadata_columns: ["original_id"], -// }; -// }; - -// // main export — call this from llmHelpers -// export const extractSubjectAnalysis = (allFiles: string[]): SubjectAnalysis => { -// const fromDir = extractFromDirectoryStructure(allFiles); -// const base = fromDir ?? -// extractFromFlatFilenames(allFiles) ?? { -// success: false, -// method: "none", -// subject_records: [], -// subject_count: 0, -// has_site_info: false, -// variants_by_subject: {}, -// python_generated_filename_rules: [], -// }; - -// const idMapping = generateIdMapping(base); -// return { ...base, id_mapping: idMapping }; -// }; diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts index 4708c13..24aa930 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts +++ b/src/components/User/Dashboard/DatasetOrganizer/utils/filenameTokenizer.ts @@ -1,6 +1,6 @@ // src/components/DatasetOrganizer/utils/filenameTokenizer.ts -// Port of autobidsify's filename_tokenizer.py -// Philosophy: Python stats → dominant prefixes → subject IDs (no LLM needed for this part) +// Mirrors filename_tokenizer.py + export interface SubjectRecord { original_id: string; numeric_id: string; @@ -52,40 +52,21 @@ const COMMON_WORDS = new Set([ "experiment", ]); +const DATA_EXTENSIONS = + /\.(snirf|nii|nii\.gz|dcm|mat|nirs|jnii|bnii|h5|hdf5|edf|bdf)$/i; + // ============================================================================ -// FilenameTokenizer — mirrors FilenameTokenizer class in filename_tokenizer.py +// FilenamePatternAnalyzer — mirrors FilenameTokenizer class // ============================================================================ - -/** - * Advanced split: CamelCase + number boundaries - * "VHMCT" → ["VHM", "CT"] - * "CT1mm" → ["CT", "1", "mm"] - * "sub82352" → ["sub", "82352"] - */ -const splitAdvanced = (text: string): string[] => { - if (!text) return []; - - // Keep known neuroimaging terms together - if (NEUROIMAGING_TERMS.has(text)) return [text]; - - // Split on type boundaries: - // - Uppercase sequence before uppercase+lowercase: "VHM" before "CT" - // - CamelCase: uppercase followed by lowercase - // - Letter/digit boundaries - const pattern = /([A-Z]+(?=[A-Z][a-z]|\b|[0-9])|[A-Z][a-z]+|[a-z]+|[0-9]+)/g; - const tokens = text.match(pattern) || []; - return tokens.filter((t) => t.length > 0); -}; - -/** +/* * Tokenize a filename into meaningful tokens. - * Mirrors FilenameTokenizer.tokenize() in filename_tokenizer.py * * Examples: * "VHMCT1mm-Hip (134).dcm" → ["VHM", "CT", "1", "mm", "Hip", "134"] * "Beijing_sub82352" → ["Beijing", "sub", "82352"] * "scan_001_T1w.nii" → ["scan", "001", "T1w"] */ + export const tokenizeFilename = (filename: string): string[] => { // Step 1: Remove all extensions (up to 6 chars) let name = filename; @@ -111,28 +92,60 @@ export const tokenizeFilename = (filename: string): string[] => { return tokens.filter((t) => t.trim().length >= 1); }; +/* + * Advanced split: CamelCase + number boundaries + * "VHMCT" → ["VHM", "CT"] + * "CT1mm" → ["CT", "1", "mm"] + * "sub82352" → ["sub", "82352"] + */ +const splitAdvanced = (text: string): string[] => { + if (!text) return []; + + // Keep known neuroimaging terms together + if (NEUROIMAGING_TERMS.has(text)) return [text]; + + // Split on type boundaries: + // - Uppercase sequence before uppercase+lowercase: "VHM" before "CT" + // - CamelCase: uppercase followed by lowercase + // - Letter/digit boundaries + const pattern = /([A-Z]+(?=[A-Z][a-z]|\b|[0-9])|[A-Z][a-z]+|[a-z]+|[0-9]+)/g; + const tokens = text.match(pattern) || []; + return tokens.filter((t) => t.length > 0); +}; + // ============================================================================ // FilenamePatternAnalyzer — mirrors FilenamePatternAnalyzer class // ============================================================================ +interface TokenStatistics { + totalFiles: number; + tokenFrequency: Record; + prefixFrequency: Record; + dominantPrefixes: DominantPrefix[]; + tokenPositions: Record>; // NEW — mirrors token_positions + insights: string[]; // NEW — mirrors _generate_insights() + uniqueTokenCount: number; // NEW + uniquePrefixCount: number; // NEW +} + interface DominantPrefix { prefix: string; count: number; percentage: number; } -interface TokenStatistics { - totalFiles: number; - tokenFrequency: Record; - prefixFrequency: Record; - dominantPrefixes: DominantPrefix[]; +interface LLMPayload { + task: string; + statistics: TokenStatistics; + filenameSamples: string[]; + userHints: Record; + instructions: string; } -/** - * Find dominant prefixes — tokens appearing in >5% of files - * that are not common words. - * Mirrors FilenamePatternAnalyzer._find_dominant_prefixes() - */ +// ───────────────────────────────────────────────────────────────────────────── +// Mirrors FilenamePatternAnalyzer._find_dominant_prefixes() +// ───────────────────────────────────────────────────────────────────────────── + const findDominantPrefixes = ( prefixCounter: Record, totalFiles: number @@ -154,18 +167,109 @@ const findDominantPrefixes = ( })); }; -/** - * Analyze token statistics across all filenames. - * Mirrors FilenamePatternAnalyzer.analyze_token_statistics() - */ +// ───────────────────────────────────────────────────────────────────────────── +// _generate_insights() +// Mirrors FilenamePatternAnalyzer._generate_insights() +// ───────────────────────────────────────────────────────────────────────────── + +const generateInsights = ( + allTokens: Record, + prefixTokens: Record, + dominantPrefixes: DominantPrefix[] +): string[] => { + const insights: string[] = []; + const uniqueTokenCount = Object.keys(allTokens).length; + + // Insight 1: token diversity + if (uniqueTokenCount < 20) { + insights.push( + `Low token diversity: only ${uniqueTokenCount} unique tokens across all files` + ); + } else if (uniqueTokenCount > 100) { + insights.push( + `High token diversity: ${uniqueTokenCount} unique tokens detected` + ); + } + + // Insight 2: prefix distribution + if (dominantPrefixes.length === 0) { + insights.push("No dominant filename prefixes detected"); + } else if (dominantPrefixes.length === 1) { + const p = dominantPrefixes[0]; + insights.push( + `Single dominant prefix '${p.prefix}' in ${p.percentage}% of files` + ); + } else if (dominantPrefixes.length === 2) { + const [p1, p2] = dominantPrefixes; + insights.push( + `Two major prefixes detected: '${p1.prefix}' (${p1.percentage}%) and '${p2.prefix}' (${p2.percentage}%)` + ); + } else { + insights.push( + `${dominantPrefixes.length} dominant prefixes detected, suggesting possible subject groupings` + ); + } + + // Insight 3: most common tokens + const topTokens = Object.entries(allTokens) + .sort((a, b) => b[1] - a[1]) + .slice(0, 3); + if (topTokens.length > 0) { + const commonList = topTokens.map(([t, c]) => `'${t}' (${c})`).join(", "); + insights.push(`Most frequent tokens: ${commonList}`); + } + + return insights; +}; + +// ───────────────────────────────────────────────────────────────────────────── +// _sample_diverse_filenames() +// Mirrors FilenamePatternAnalyzer._sample_diverse_filenames() +// ───────────────────────────────────────────────────────────────────────────── + +const sampleDiverseFilenames = ( + filenames: string[], + maxSamples: number = 30 +): string[] => { + if (filenames.length <= maxSamples) return [...filenames].sort(); + + // Group by first token (mirrors Python: prefix_groups[prefix].append(filename)) + const prefixGroups: Record = {}; + for (const filename of filenames) { + const tokens = tokenizeFilename(filename); + const prefix = tokens.length > 0 ? tokens[0] : "none"; + if (!prefixGroups[prefix]) prefixGroups[prefix] = []; + prefixGroups[prefix].push(filename); + } + + const groupCount = Object.keys(prefixGroups).length; + const samplesPerGroup = Math.max(1, Math.floor(maxSamples / groupCount)); + + const samples: string[] = []; + for (const prefix of Object.keys(prefixGroups).sort()) { + const groupFiles = prefixGroups[prefix]; + const n = Math.min(groupFiles.length, samplesPerGroup); + samples.push(...[...groupFiles].sort().slice(0, n)); + if (samples.length >= maxSamples) break; + } + + return samples.slice(0, maxSamples); +}; + +// ───────────────────────────────────────────────────────────────────────────── +// analyze_token_statistics() +// Mirrors FilenamePatternAnalyzer.analyze_token_statistics() +// ───────────────────────────────────────────────────────────────────────────── + export const analyzeTokenStatistics = ( filenames: string[] ): TokenStatistics => { const allTokens: Record = {}; - const prefixTokens: Record = {}; // first token only + const prefixTokens: Record = {}; + const positionTokens: Record> = {}; for (const filename of filenames) { - // Extract just filename from path + // Mirror Python __init__: strip to just filename if path provided const fname = filename.includes("/") ? filename.split("/").pop()! : filename; @@ -177,23 +281,331 @@ export const analyzeTokenStatistics = ( allTokens[token] = (allTokens[token] || 0) + 1; } - // CRITICAL: use first TOKEN as prefix (not regex match) + // CRITICAL: use first TOKEN as prefix (not regex) if (tokens.length > 0) { const firstToken = tokens[0]; prefixTokens[firstToken] = (prefixTokens[firstToken] || 0) + 1; } + + // NEW: count tokens by position — mirrors position_tokens[i][token] += 1 + tokens.forEach((token, i) => { + if (!positionTokens[i]) positionTokens[i] = {}; + positionTokens[i][token] = (positionTokens[i][token] || 0) + 1; + }); + } + + // Cap frequencies — mirrors .most_common(50) / .most_common(20) + const tokenFrequency = Object.fromEntries( + Object.entries(allTokens) + .sort((a, b) => b[1] - a[1]) + .slice(0, 50) + ); + const prefixFrequency = Object.fromEntries( + Object.entries(prefixTokens) + .sort((a, b) => b[1] - a[1]) + .slice(0, 20) + ); + + // Cap each position bucket at top 10 — mirrors .most_common(10) + const tokenPositions: Record> = {}; + for (const [pos, counter] of Object.entries(positionTokens)) { + tokenPositions[Number(pos)] = Object.fromEntries( + Object.entries(counter) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + ); } const dominantPrefixes = findDominantPrefixes(prefixTokens, filenames.length); + const insights = generateInsights(allTokens, prefixTokens, dominantPrefixes); return { totalFiles: filenames.length, - tokenFrequency: allTokens, - prefixFrequency: prefixTokens, + tokenFrequency, + prefixFrequency, dominantPrefixes, + tokenPositions, // NEW + insights, // NEW + uniqueTokenCount: Object.keys(allTokens).length, // NEW + uniquePrefixCount: Object.keys(prefixTokens).length, // NEW + }; +}; + +// ───────────────────────────────────────────────────────────────────────────── +// build_llm_payload() +// Mirrors FilenamePatternAnalyzer.build_llm_payload() +// ───────────────────────────────────────────────────────────────────────────── + +export const buildLLMPayload = ( + filenames: string[], + userHints: Record, + maxSamples: number = 30 +): LLMPayload => { + const stats = analyzeTokenStatistics(filenames); + const filenameSamples = sampleDiverseFilenames(filenames, maxSamples); + + return { + task: "subject_identification", + statistics: stats, + filenameSamples, + userHints, + instructions: + "Analyze the filename token statistics and samples. " + + "Determine how to group files by subject. " + + "The 'dominant_prefixes' may indicate subject identifiers. " + + "The 'insights' provide observations. " + + "User hint 'n_subjects' can help validate your hypothesis.", + }; +}; + +/** + * Analyze token statistics across all filenames. + * Mirrors FilenamePatternAnalyzer.analyze_token_statistics() + */ +// export const analyzeTokenStatistics = ( +// filenames: string[] +// ): TokenStatistics => { +// const allTokens: Record = {}; +// const prefixTokens: Record = {}; // first token only + +// for (const filename of filenames) { +// // Extract just filename from path +// const fname = filename.includes("/") +// ? filename.split("/").pop()! +// : filename; + +// const tokens = tokenizeFilename(fname); + +// // Count all tokens +// for (const token of tokens) { +// allTokens[token] = (allTokens[token] || 0) + 1; +// } + +// // CRITICAL: use first TOKEN as prefix (not regex match) +// if (tokens.length > 0) { +// const firstToken = tokens[0]; +// prefixTokens[firstToken] = (prefixTokens[firstToken] || 0) + 1; +// } +// } + +// const dominantPrefixes = findDominantPrefixes(prefixTokens, filenames.length); + +// return { +// totalFiles: filenames.length, +// tokenFrequency: allTokens, +// prefixFrequency: prefixTokens, +// dominantPrefixes, +// }; +// }; + +/* + * Find dominant prefixes — tokens appearing in >5% of files + * that are not common words. + * Mirrors FilenamePatternAnalyzer._find_dominant_prefixes() + */ +// const findDominantPrefixes = ( +// prefixCounter: Record, +// totalFiles: number +// ): DominantPrefix[] => { +// const threshold = totalFiles * 0.05; // 5% threshold + +// return Object.entries(prefixCounter) +// .filter(([prefix, count]) => { +// if (count < threshold) return false; +// if (COMMON_WORDS.has(prefix.toLowerCase())) return false; +// return true; +// }) +// .sort((a, b) => b[1] - a[1]) +// .slice(0, 20) +// .map(([prefix, count]) => ({ +// prefix, +// count, +// percentage: Math.round((count / totalFiles) * 1000) / 10, +// })); +// }; + +// ============================================================================ +// Integration Functions +// Mirrors analyze_filenames_for_subjects() + _generate_recommendation() in filename_tokenizer.py +// ============================================================================ + +export interface FilenameAnalysisResult { + python_statistics: TokenStatistics; + llm_payload: LLMPayload; + confidence: "high" | "medium" | "low" | "none"; + recommendation: string; +} + +/** + * Main entry point: analyze filenames to detect subject groupings. + * Mirrors analyze_filenames_for_subjects() in filename_tokenizer.py + * + * Called from buildEvidenceBundle() in llmHelpers.ts — replaces the + * manual filenameAnalysis block that was built inline there. + */ +export const analyzeFilenamesForSubjects = ( + allFiles: string[], + userHints: Record +): FilenameAnalysisResult => { + // Mirror Python: extract just filenames, not full paths + const filenames = allFiles.map((f) => + f.includes("/") ? f.split("/").pop()! : f + ); + + const stats = analyzeTokenStatistics(filenames); + const llmPayload = buildLLMPayload(filenames, userHints, 30); + + // Assess confidence — mirrors Python confidence logic exactly + const dominantCount = stats.dominantPrefixes.length; + const userNSubjects: number | null = userHints?.n_subjects ?? null; + + let confidence: "high" | "medium" | "low" | "none" = "none"; + if (dominantCount > 0) { + if (userNSubjects && dominantCount === userNSubjects) { + confidence = "high"; + } else if (dominantCount >= 2 && dominantCount <= 10) { + confidence = "medium"; + } else { + confidence = "low"; + } + } + + const recommendation = generateRecommendation(stats, userHints); + + return { + python_statistics: stats, + llm_payload: llmPayload, + confidence, + recommendation, }; }; +/** + * Mirrors _generate_recommendation() in filename_tokenizer.py + */ +const generateRecommendation = ( + stats: TokenStatistics, + userHints: Record +): string => { + const dominantPrefixes = stats.dominantPrefixes; + const userNSubjects: number | null = userHints?.n_subjects ?? null; + + if (dominantPrefixes.length === 0) { + return ( + "No clear filename patterns detected. " + + "Recommend using --describe to explain subject identification." + ); + } + + if (userNSubjects && dominantPrefixes.length === userNSubjects) { + const prefixesStr = dominantPrefixes.map((p) => p.prefix).join(", "); + return ( + `HIGH CONFIDENCE: Detected ${dominantPrefixes.length} dominant prefixes ` + + `(${prefixesStr}) matching user hint of ${userNSubjects} subjects.` + ); + } + + if (dominantPrefixes.length >= 2 && dominantPrefixes.length <= 5) { + return ( + `MEDIUM CONFIDENCE: Detected ${dominantPrefixes.length} potential subject groups. ` + + `Will send to LLM for validation.` + ); + } + + return ( + `LOW CONFIDENCE: Found ${dominantPrefixes.length} prefix patterns, ` + + `which may or may not represent subjects. LLM will analyze.` + ); +}; + +// ============================================================================ +// SubjectGroupingDecision +// Mirrors SubjectGroupingDecision class in filename_tokenizer.py +// Not used in runtime flow — used as typed helpers when parsing LLM responses +// ============================================================================ + +export interface PrefixMappingDecision { + method: "prefix_based"; + description: string; + rules: Array<{ + prefix: string; + maps_to_subject: string; + match_pattern: string; + }>; + participant_metadata: Record>; +} + +export interface SequentialAssignmentDecision { + method: "sequential"; + n_subjects: number; + note: string; +} + +export interface BlockingQuestionDecision { + method: "blocked"; + reason: string; + question: { + type: string; + severity: string; + message: string; + options: string[]; + }; +} + +export type SubjectGroupingDecision = + | PrefixMappingDecision + | SequentialAssignmentDecision + | BlockingQuestionDecision; + +/** + * Mirrors SubjectGroupingDecision.create_prefix_mapping() + */ +export const createPrefixMapping = ( + prefixToSubject: Record, + metadata?: Record> +): PrefixMappingDecision => ({ + method: "prefix_based", + description: `Files grouped by ${ + Object.keys(prefixToSubject).length + } filename prefixes`, + rules: Object.entries(prefixToSubject).map(([prefix, subjId]) => ({ + prefix, + maps_to_subject: subjId, + match_pattern: `${prefix}*`, + })), + participant_metadata: metadata ?? {}, +}); + +/** + * Mirrors SubjectGroupingDecision.create_sequential_assignment() + */ +export const createSequentialAssignment = ( + nSubjects: number +): SequentialAssignmentDecision => ({ + method: "sequential", + n_subjects: nSubjects, + note: + "No clear subject grouping pattern detected in filenames. " + + "Assigning sequential IDs based on file order or user hint.", +}); + +/** + * Mirrors SubjectGroupingDecision.create_blocking_question() + */ +export const createBlockingQuestion = ( + reason: string, + options: string[] +): BlockingQuestionDecision => ({ + method: "blocked", + reason, + question: { + type: "subject_grouping", + severity: "block", + message: reason, + options, + }, +}); + // ============================================================================ // extractSubjectAnalysis — mirrors build_bids_plan()'s subject extraction // ============================================================================ @@ -495,7 +907,6 @@ const extractFromDirectoryStructure = ( }; }; -const DATA_EXTENSIONS = /\.(snirf|nii|nii\.gz|dcm|mat|nirs|h5|hdf5|edf|bdf)$/i; const TRIO_FILENAMES = new Set([ "dataset_description.json", "participants.tsv", diff --git a/src/components/User/Dashboard/DatasetOrganizer/utils/plannerHelpers.ts b/src/components/User/Dashboard/DatasetOrganizer/utils/plannerHelpers.ts new file mode 100644 index 0000000..e69de29 From edb2a8f9b4291eb30e91a79dbefad237e123378b Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 3 Apr 2026 17:40:39 -0400 Subject: [PATCH 06/61] feat: reorganize the code for integrated autobidsify --- package.json | 1 + .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 1384 ++++++++--------- .../DatasetOrganizer/utils/fileAnalyzers.ts | 205 ++- .../DatasetOrganizer/utils/fileProcessors.ts | 68 +- .../utils/filenameTokenizer.ts | 588 ------- .../Dashboard/DatasetOrganizer/utils/llm.ts | 963 ++++++++++++ .../DatasetOrganizer/utils/llmHelpers.ts | 983 +++++++----- .../DatasetOrganizer/utils/llmPrompts.ts | 619 +------- .../DatasetOrganizer/utils/plannerHelpers.ts | 870 +++++++++++ .../DatasetOrganizer/utils/trioHelpers.ts | 762 +++++++++ src/services/ollama.service.ts | 17 +- yarn.lock | 5 + 12 files changed, 4221 insertions(+), 2244 deletions(-) create mode 100644 src/components/User/Dashboard/DatasetOrganizer/utils/llm.ts create mode 100644 src/components/User/Dashboard/DatasetOrganizer/utils/trioHelpers.ts diff --git a/package.json b/package.json index 1f8144e..6f414ef 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "devDependencies": { "@babel/plugin-proposal-private-property-in-object": "^7.21.11", "@trivago/prettier-plugin-sort-imports": "^4.2.0", + "@types/js-yaml": "^4.0.9", "@types/node": "^20.5.7", "@types/pako": "^2.0.3", "@typescript-eslint/eslint-plugin": "^5.31.0", diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index a8fd052..9658ec6 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -1,23 +1,12 @@ import { generateId } from "./utils/fileProcessors"; -import { extractSubjectAnalysis } from "./utils/filenameTokenizer"; -//add +import { LLMConfig } from "./utils/llm"; import { - buildFileSummary, - analyzeFilePatterns, - getUserContext, - getFileAnnotations, - downloadJSON, buildEvidenceBundle, - extractSubjectsFromFiles, buildIngestInfo, + downloadJSON, } from "./utils/llmHelpers"; -import { - getDatasetDescriptionPrompt, - getReadmePrompt, - getParticipantsPrompt, - getConversionScriptPrompt, - getBIDSPlanPrompt, -} from "./utils/llmPrompts"; +import { buildBidsPlan } from "./utils/plannerHelpers"; +import { generateTrioFiles } from "./utils/trioHelpers"; import { Close, ContentCopy, @@ -40,10 +29,12 @@ import { Alert, } from "@mui/material"; import { Colors } from "design/theme"; +import { dump as yamlDump } from "js-yaml"; import JSZip from "jszip"; import React, { useState, useEffect } from "react"; import { FileItem } from "redux/projects/types/projects.interface"; -import { OllamaService } from "services/ollama.service"; + +// import { OllamaService } from "services/ollama.service"; interface LLMPanelProps { files: FileItem[]; @@ -158,6 +149,16 @@ const LLMPanel: React.FC = ({ const [panelHeight, setPanelHeight] = useState(450); const [isResizing, setIsResizing] = useState(false); + // Build LLMConfig for all helper calls — mirrors autobidsify CLI arg assembly + const buildLLMConfig = (): LLMConfig => ({ + provider, + model, + apiKey, + baseUrl: currentProvider.baseUrl, + isAnthropic: currentProvider.isAnthropic, + noApiKey: currentProvider.noApiKey, + }); + // ======================================================================== // BUTTON 1: GENERATE EVIDENCE BUNDLE // ======================================================================== @@ -202,406 +203,27 @@ const LLMPanel: React.FC = ({ setError("Please generate evidence bundle first"); return; } - if (!currentProvider.noApiKey && !apiKey.trim()) { setError("Please enter an API key"); return; } - // Create abort controller const controller = new AbortController(); setAbortController(controller); - setGeneratingTrio(true); setError(null); setStatus("Generating BIDS trio files..."); try { - const userText = evidenceBundle.user_hints.user_text || ""; - - // ========================================== - // Call 1: Generate dataset_description.json - // ========================================== - let datasetDesc: any; - if (evidenceBundle.trio_found?.["dataset_description.json"]) { - setStatus("1/3 dataset_description.json already exists, skipping..."); - const existing = files.find( - (f) => f.source === "user" && f.name === "dataset_description.json" - ); - datasetDesc = existing?.content ? JSON.parse(existing.content) : {}; - } else { - setStatus("1/3 Generating dataset_description.json..."); - const ddPrompt = getDatasetDescriptionPrompt(userText, evidenceBundle); - - let ddResponse; - if (currentProvider.isAnthropic) { - ddResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: ddPrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // ddResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: ddPrompt }], - // stream: false, - // }), - // }); - ddResponse = await OllamaService.chat(model, [ - { role: "user", content: ddPrompt }, - ]); - } else { - ddResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: ddPrompt }], - max_tokens: 2048, - }), - }); - } - - // const ddData = await ddResponse.json(); - const ddData = - provider === "ollama" ? ddResponse : await ddResponse.json(); - let ddText = currentProvider.isAnthropic - ? ddData.content[0].text - : ddData.choices[0].message.content; - - // Clean up markdown fences - ddText = ddText - .replace(/^```json\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); - datasetDesc = JSON.parse(ddText); - } - - // ========================================== - // Call 2: Generate README.md - // ========================================== - let readmeContent: string; - if (evidenceBundle.trio_found?.["README.md"]) { - setStatus("2/3 README.md already exists, skipping..."); - const existing = files.find( - (f) => - f.source === "user" && - ["README.md", "README.txt", "README.rst", "readme.md"].includes( - f.name - ) - ); - readmeContent = existing?.content || ""; - } else { - setStatus("2/3 Generating README.md..."); - const readmePrompt = getReadmePrompt(userText); - - let readmeResponse; - if (currentProvider.isAnthropic) { - readmeResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: readmePrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // readmeResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: readmePrompt }], - // stream: false, - // }), - // }); - readmeResponse = await OllamaService.chat(model, [ - { role: "user", content: readmePrompt }, - ]); - } else { - readmeResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: readmePrompt }], - max_tokens: 2048, - }), - }); - } - - // const readmeData = await readmeResponse.json(); - const readmeData = - provider === "ollama" ? readmeResponse : await readmeResponse.json(); - readmeContent = currentProvider.isAnthropic - ? readmeData.content[0].text - : readmeData.choices[0].message.content; - } - // ========================================== - // Call 3: Generate participants.tsv - // ========================================== - let participantsContent: string; - if (evidenceBundle.trio_found?.["participants.tsv"]) { - setStatus("3/3 participants.tsv already exists, skipping..."); - const existing = files.find( - (f) => f.source === "user" && f.name === "participants.tsv" - ); - participantsContent = existing?.content || ""; - } else { - setStatus("3/3 Generating participants.tsv..."); - const partsPrompt = getParticipantsPrompt(userText); - - // ← ADD HERE: compute subject analysis before try block so it's in scope - const currentSubjectAnalysis = extractSubjectAnalysis( - evidenceBundle?.all_files || [], - evidenceBundle?.user_hints?.n_subjects, - evidenceBundle?.filename_analysis?.python_statistics - ?.dominant_prefixes - ); + const { datasetDesc, readmeContent, participantsTsv, skipped } = + await generateTrioFiles({ + evidenceBundle, + files, + llmConfig: buildLLMConfig(), + signal: controller.signal, + onStatus: setStatus, + }); - console.log("=== PARTICIPANTS DEBUG ==="); - console.log("method:", currentSubjectAnalysis?.method); - console.log("subject_count:", currentSubjectAnalysis?.subject_count); - console.log( - "id_mapping:", - currentSubjectAnalysis?.id_mapping?.id_mapping - ); - console.log( - "reverse_mapping:", - currentSubjectAnalysis?.id_mapping?.reverse_mapping - ); - console.log( - "subject_records sample:", - currentSubjectAnalysis?.subject_records?.slice(0, 3) - ); - const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; - const expectedCount = evidenceBundle?.user_hints?.n_subjects; - const subjectLabels: string[] = - idMap && - Object.keys(idMap).length > 0 && - (!expectedCount || Object.keys(idMap).length === expectedCount) - ? Object.values(idMap).map((id: string) => `sub-${id}`) - : Array.from( - { - length: expectedCount || Object.keys(idMap || {}).length || 1, - }, - (_, i) => `sub-${String(i + 1).padStart(2, "0")}` - ); - - let partsResponse; - if (currentProvider.isAnthropic) { - partsResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 1024, - messages: [{ role: "user", content: partsPrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // partsResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: partsPrompt }], - // stream: false, - // }), - // }); - partsResponse = await OllamaService.chat(model, [ - { role: "user", content: partsPrompt }, - ]); - } else { - partsResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: partsPrompt }], - max_tokens: 1024, - }), - }); - } - - // const partsData = await partsResponse.json(); - const partsData = - provider === "ollama" ? partsResponse : await partsResponse.json(); - const participantsRaw = currentProvider.isAnthropic - ? partsData.content[0].text - : partsData.choices[0].message.content; - - // Build TSV from schema - // try { - // const schemaText = participantsRaw - // .replace(/^```json\n?/g, "") - // .replace(/\n?```$/g, "") - // .trim(); - // const schema = JSON.parse(schemaText); - // const columns: string[] = schema.columns.map((c: any) => c.name); - - // // Get subject IDs from evidence bundle (extracted by Python-style analysis) - // // const idMapping = - // // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; - // // const subjectLabels: string[] = idMapping - // // ? Object.values(idMapping).map((id) => `sub-${id}`) - // // : ["sub-01"]; // fallback if no subject analysis - // // Get subject IDs from subjectAnalysis state (computed at plan stage) - // // Fall back to computing fresh if plan hasn't been run yet - // const currentSubjectAnalysis = - // subjectAnalysis || - // extractSubjectAnalysis( - // evidenceBundle?.all_files || [], - // evidenceBundle?.user_hints?.n_subjects, - // evidenceBundle?.filename_analysis?.python_statistics - // ?.dominant_prefixes - // ); - // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; - // const subjectLabels: string[] = - // idMap && Object.keys(idMap).length > 0 - // ? Object.values(idMap).map((id) => `sub-${id}`) - // : Array.from( - // { length: evidenceBundle?.user_hints?.n_subjects || 1 }, - // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` - // ); - - // const header = columns.join("\t"); - // // ====origin==== - // // const rows = subjectLabels.map((subId) => - // // columns - // // .map((col: string) => (col === "participant_id" ? subId : "n/a")) - // // .join("\t") - // // ); - // //====== end ====== - // // =====update start===== - // const reverseMap = - // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; - // const subjectRecords = currentSubjectAnalysis?.subject_records || []; - - // const rows = subjectLabels.map((subId) => { - // const bareId = subId.replace(/^sub-/, ""); - // const originalId = reverseMap[bareId]; - // const record = subjectRecords.find( - // (r: any) => r.original_id === originalId - // ); - // return columns - // .map((col: string) => { - // if (col === "participant_id") return subId; - // if (col === "original_id") return originalId || "n/a"; - // if (col === "group") return (record as any)?.group || "n/a"; - // return "n/a"; - // }) - // .join("\t"); - // }); - // //====update end====== - // participantsContent = [header, ...rows].join("\n"); - // } catch (e) { - // // Fallback: LLM didn't return valid JSON schema, use raw content - // participantsContent = participantsRaw - // .replace(/^```\n?/g, "") - // .replace(/\n?```$/g, "") - // .trim(); - // } - // Build TSV from schema + subject analysis - // Mirrors _generate_participants_tsv_from_python() in planner.py - try { - const schemaText = participantsRaw - .replace(/^```json\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); - const schema = JSON.parse(schemaText); - - // LLM decides extra demographic columns (sex, age, group etc.) - // but we always add participant_id and original_id ourselves - const extraColumns: string[] = schema.columns - .map((c: any) => c.name) - .filter( - (name: string) => - name !== "participant_id" && name !== "original_id" - ); - - // Always start with participant_id and original_id - const columns = ["participant_id", "original_id", ...extraColumns]; - - const reverseMap = - currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; - const subjectRecords = currentSubjectAnalysis?.subject_records || []; - - const header = columns.join("\t"); - const rows = subjectLabels.map((subId) => { - const bareId = subId.replace(/^sub-/, ""); - const originalId = reverseMap[bareId] || "n/a"; - const record = subjectRecords.find( - (r: any) => r.original_id === originalId - ); - return columns - .map((col: string) => { - if (col === "participant_id") return subId; - if (col === "original_id") return originalId; - if (col === "group") return (record as any)?.group || "n/a"; - return "n/a"; - }) - .join("\t"); - }); - - participantsContent = [header, ...rows].join("\n"); - } catch (e) { - // Fallback: generate minimal TSV directly from subject analysis - const reverseMap = - currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; - const header = "participant_id\toriginal_id"; - const rows = subjectLabels.map((subId) => { - const bareId = subId.replace(/^sub-/, ""); - const originalId = reverseMap[bareId] || "n/a"; - return `${subId}\t${originalId}`; - }); - participantsContent = [header, ...rows].join("\n"); - } - } - // ========================================== - // Add trio files to Virtual File System - // ========================================== const timestamp = new Date().toLocaleString(); const trioFiles: FileItem[] = [ { @@ -621,10 +243,7 @@ const LLMPanel: React.FC = ({ name: "README.md", type: "file", fileType: "meta", - content: readmeContent - .replace(/^```markdown\n?/g, "") - .replace(/\n?```$/g, "") - .trim(), + content: readmeContent, contentType: "text", isUserMeta: true, parentId: null, @@ -636,10 +255,7 @@ const LLMPanel: React.FC = ({ name: "participants.tsv", type: "file", fileType: "meta", - content: participantsContent - .replace(/^```\n?/g, "") - .replace(/\n?```$/g, "") - .trim(), + content: participantsTsv, contentType: "text", isUserMeta: true, parentId: null, @@ -647,32 +263,27 @@ const LLMPanel: React.FC = ({ generatedAt: timestamp, }, ]; - // replace existing trio files, add if not exist + updateFiles((prev) => { const trioNames = [ "dataset_description.json", "README.md", "participants.tsv", ]; - - // Remove old AI generated trio files const withoutOldTrio = prev.filter( (f) => !(f.source === "ai" && trioNames.includes(f.name)) ); - - // Add new trio files - // return [...withoutOldTrio, ...trioFiles]; - - // Only add AI-generated files for ones that weren't user-uploaded - const newTrioFiles = trioFiles.filter( - (tf) => - !evidenceBundle.trio_found?.[ - tf.name as keyof typeof evidenceBundle.trio_found - ] - ); - + // Only add AI files for ones that weren't user-uploaded (skipped=true means user-uploaded) + const newTrioFiles = trioFiles.filter((tf) => { + if (tf.name === "dataset_description.json") + return !skipped.datasetDesc; + if (tf.name === "README.md") return !skipped.readme; + if (tf.name === "participants.tsv") return !skipped.participants; + return true; + }); return [...withoutOldTrio, ...newTrioFiles]; }); + setTrioGenerated(true); setStatus( "✓ BIDS trio files generated and added to Virtual File System!" @@ -686,9 +297,469 @@ const LLMPanel: React.FC = ({ } } finally { setGeneratingTrio(false); - setAbortController(null); // Clear controller + setAbortController(null); } }; + // const handleGenerateTrio = async () => { + // if (!evidenceBundle) { + // setError("Please generate evidence bundle first"); + // return; + // } + + // if (!currentProvider.noApiKey && !apiKey.trim()) { + // setError("Please enter an API key"); + // return; + // } + + // // Create abort controller + // const controller = new AbortController(); + // setAbortController(controller); + + // setGeneratingTrio(true); + // setError(null); + // setStatus("Generating BIDS trio files..."); + + // try { + // const userText = evidenceBundle.user_hints.user_text || ""; + + // // ========================================== + // // Call 1: Generate dataset_description.json + // // ========================================== + // let datasetDesc: any; + // if (evidenceBundle.trio_found?.["dataset_description.json"]) { + // setStatus("1/3 dataset_description.json already exists, skipping..."); + // const existing = files.find( + // (f) => f.source === "user" && f.name === "dataset_description.json" + // ); + // datasetDesc = existing?.content ? JSON.parse(existing.content) : {}; + // } else { + // setStatus("1/3 Generating dataset_description.json..."); + // const ddPrompt = getDatasetDescriptionPrompt(userText, evidenceBundle); + + // let ddResponse; + // if (currentProvider.isAnthropic) { + // ddResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: ddPrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // ddResponse = await OllamaService.chat(model, [ + // { role: "user", content: ddPrompt }, + // ]); + // } else { + // ddResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: ddPrompt }], + // max_tokens: 2048, + // }), + // }); + // } + + // // const ddData = await ddResponse.json(); + // const ddData = + // provider === "ollama" ? ddResponse : await ddResponse.json(); + // let ddText = currentProvider.isAnthropic + // ? ddData.content[0].text + // : ddData.choices[0].message.content; + + // // Clean up markdown fences + // ddText = ddText + // .replace(/^```json\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // datasetDesc = JSON.parse(ddText); + // } + + // // ========================================== + // // Call 2: Generate README.md + // // ========================================== + // let readmeContent: string; + // if (evidenceBundle.trio_found?.["README.md"]) { + // setStatus("2/3 README.md already exists, skipping..."); + // const existing = files.find( + // (f) => + // f.source === "user" && + // ["README.md", "README.txt", "README.rst", "readme.md"].includes( + // f.name + // ) + // ); + // readmeContent = existing?.content || ""; + // } else { + // setStatus("2/3 Generating README.md..."); + // const readmePrompt = getReadmePrompt(userText); + + // let readmeResponse; + // if (currentProvider.isAnthropic) { + // readmeResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: readmePrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // readmeResponse = await OllamaService.chat(model, [ + // { role: "user", content: readmePrompt }, + // ]); + // } else { + // readmeResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: readmePrompt }], + // max_tokens: 2048, + // }), + // }); + // } + + // const readmeData = + // provider === "ollama" ? readmeResponse : await readmeResponse.json(); + // readmeContent = currentProvider.isAnthropic + // ? readmeData.content[0].text + // : readmeData.choices[0].message.content; + // } + // // ========================================== + // // Call 3: Generate participants.tsv + // // ========================================== + // let participantsContent: string; + // if (evidenceBundle.trio_found?.["participants.tsv"]) { + // setStatus("3/3 participants.tsv already exists, skipping..."); + // const existing = files.find( + // (f) => f.source === "user" && f.name === "participants.tsv" + // ); + // participantsContent = existing?.content || ""; + // } else { + // setStatus("3/3 Generating participants.tsv..."); + // const partsPrompt = getParticipantsPrompt(userText); + + // const currentSubjectAnalysis = extractSubjectAnalysis( + // evidenceBundle?.all_files || [], + // evidenceBundle?.user_hints?.n_subjects, + // evidenceBundle?.filename_analysis?.python_statistics + // ?.dominant_prefixes + // ); + + // console.log("=== PARTICIPANTS DEBUG ==="); + // console.log("method:", currentSubjectAnalysis?.method); + // console.log("subject_count:", currentSubjectAnalysis?.subject_count); + // console.log( + // "id_mapping:", + // currentSubjectAnalysis?.id_mapping?.id_mapping + // ); + // console.log( + // "reverse_mapping:", + // currentSubjectAnalysis?.id_mapping?.reverse_mapping + // ); + // console.log( + // "subject_records sample:", + // currentSubjectAnalysis?.subject_records?.slice(0, 3) + // ); + // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + // const expectedCount = evidenceBundle?.user_hints?.n_subjects; + // const subjectLabels: string[] = + // idMap && + // Object.keys(idMap).length > 0 && + // (!expectedCount || Object.keys(idMap).length === expectedCount) + // ? Object.values(idMap).map((id: string) => `sub-${id}`) + // : Array.from( + // { + // length: expectedCount || Object.keys(idMap || {}).length || 1, + // }, + // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + // ); + + // let partsResponse; + // if (currentProvider.isAnthropic) { + // partsResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 1024, + // messages: [{ role: "user", content: partsPrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // partsResponse = await OllamaService.chat(model, [ + // { role: "user", content: partsPrompt }, + // ]); + // } else { + // partsResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: partsPrompt }], + // max_tokens: 1024, + // }), + // }); + // } + + // // const partsData = await partsResponse.json(); + // const partsData = + // provider === "ollama" ? partsResponse : await partsResponse.json(); + // const participantsRaw = currentProvider.isAnthropic + // ? partsData.content[0].text + // : partsData.choices[0].message.content; + + // // Build TSV from schema + // // try { + // // const schemaText = participantsRaw + // // .replace(/^```json\n?/g, "") + // // .replace(/\n?```$/g, "") + // // .trim(); + // // const schema = JSON.parse(schemaText); + // // const columns: string[] = schema.columns.map((c: any) => c.name); + + // // // Get subject IDs from evidence bundle (extracted by Python-style analysis) + // // // const idMapping = + // // // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; + // // // const subjectLabels: string[] = idMapping + // // // ? Object.values(idMapping).map((id) => `sub-${id}`) + // // // : ["sub-01"]; // fallback if no subject analysis + // // // Get subject IDs from subjectAnalysis state (computed at plan stage) + // // // Fall back to computing fresh if plan hasn't been run yet + // // const currentSubjectAnalysis = + // // subjectAnalysis || + // // extractSubjectAnalysis( + // // evidenceBundle?.all_files || [], + // // evidenceBundle?.user_hints?.n_subjects, + // // evidenceBundle?.filename_analysis?.python_statistics + // // ?.dominant_prefixes + // // ); + // // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + // // const subjectLabels: string[] = + // // idMap && Object.keys(idMap).length > 0 + // // ? Object.values(idMap).map((id) => `sub-${id}`) + // // : Array.from( + // // { length: evidenceBundle?.user_hints?.n_subjects || 1 }, + // // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + // // ); + + // // const header = columns.join("\t"); + // // // ====origin==== + // // // const rows = subjectLabels.map((subId) => + // // // columns + // // // .map((col: string) => (col === "participant_id" ? subId : "n/a")) + // // // .join("\t") + // // // ); + // // //====== end ====== + // // // =====update start===== + // // const reverseMap = + // // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // // const subjectRecords = currentSubjectAnalysis?.subject_records || []; + + // // const rows = subjectLabels.map((subId) => { + // // const bareId = subId.replace(/^sub-/, ""); + // // const originalId = reverseMap[bareId]; + // // const record = subjectRecords.find( + // // (r: any) => r.original_id === originalId + // // ); + // // return columns + // // .map((col: string) => { + // // if (col === "participant_id") return subId; + // // if (col === "original_id") return originalId || "n/a"; + // // if (col === "group") return (record as any)?.group || "n/a"; + // // return "n/a"; + // // }) + // // .join("\t"); + // // }); + // // //====update end====== + // // participantsContent = [header, ...rows].join("\n"); + // // } catch (e) { + // // // Fallback: LLM didn't return valid JSON schema, use raw content + // // participantsContent = participantsRaw + // // .replace(/^```\n?/g, "") + // // .replace(/\n?```$/g, "") + // // .trim(); + // // } + // // Build TSV from schema + subject analysis + // // Mirrors _generate_participants_tsv_from_python() in planner.py + // try { + // const schemaText = participantsRaw + // .replace(/^```json\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // const schema = JSON.parse(schemaText); + + // // LLM decides extra demographic columns (sex, age, group etc.) + // // but we always add participant_id and original_id ourselves + // const extraColumns: string[] = schema.columns + // .map((c: any) => c.name) + // .filter( + // (name: string) => + // name !== "participant_id" && name !== "original_id" + // ); + + // // Always start with participant_id and original_id + // const columns = ["participant_id", "original_id", ...extraColumns]; + + // const reverseMap = + // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // const subjectRecords = currentSubjectAnalysis?.subject_records || []; + + // const header = columns.join("\t"); + // const rows = subjectLabels.map((subId) => { + // const bareId = subId.replace(/^sub-/, ""); + // const originalId = reverseMap[bareId] || "n/a"; + // const record = subjectRecords.find( + // (r: any) => r.original_id === originalId + // ); + // return columns + // .map((col: string) => { + // if (col === "participant_id") return subId; + // if (col === "original_id") return originalId; + // if (col === "group") return (record as any)?.group || "n/a"; + // return "n/a"; + // }) + // .join("\t"); + // }); + + // participantsContent = [header, ...rows].join("\n"); + // } catch (e) { + // // Fallback: generate minimal TSV directly from subject analysis + // const reverseMap = + // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // const header = "participant_id\toriginal_id"; + // const rows = subjectLabels.map((subId) => { + // const bareId = subId.replace(/^sub-/, ""); + // const originalId = reverseMap[bareId] || "n/a"; + // return `${subId}\t${originalId}`; + // }); + // participantsContent = [header, ...rows].join("\n"); + // } + // } + // // ========================================== + // // Add trio files to Virtual File System + // // ========================================== + // const timestamp = new Date().toLocaleString(); + // const trioFiles: FileItem[] = [ + // { + // id: generateId(), + // name: "dataset_description.json", + // type: "file", + // fileType: "meta", + // content: JSON.stringify(datasetDesc, null, 2), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // { + // id: generateId(), + // name: "README.md", + // type: "file", + // fileType: "meta", + // content: readmeContent + // .replace(/^```markdown\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // { + // id: generateId(), + // name: "participants.tsv", + // type: "file", + // fileType: "meta", + // content: participantsContent + // .replace(/^```\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // ]; + // // replace existing trio files, add if not exist + // updateFiles((prev) => { + // const trioNames = [ + // "dataset_description.json", + // "README.md", + // "participants.tsv", + // ]; + + // // Remove old AI generated trio files + // const withoutOldTrio = prev.filter( + // (f) => !(f.source === "ai" && trioNames.includes(f.name)) + // ); + + // // Add new trio files + // // return [...withoutOldTrio, ...trioFiles]; + + // // Only add AI-generated files for ones that weren't user-uploaded + // const newTrioFiles = trioFiles.filter( + // (tf) => + // !evidenceBundle.trio_found?.[ + // tf.name as keyof typeof evidenceBundle.trio_found + // ] + // ); + + // return [...withoutOldTrio, ...newTrioFiles]; + // }); + // setTrioGenerated(true); + // setStatus( + // "✓ BIDS trio files generated and added to Virtual File System!" + // ); + // } catch (err: any) { + // if (err.name === "AbortError") { + // setStatus("❌ Generation cancelled"); + // } else { + // setError(err.message || "Failed to generate trio files"); + // setStatus("❌ Error generating trio files"); + // } + // } finally { + // setGeneratingTrio(false); + // setAbortController(null); // Clear controller + // } + // }; const handleMouseDown = (e: React.MouseEvent) => { setIsResizing(true); @@ -725,312 +796,222 @@ const LLMPanel: React.FC = ({ const currentProvider = llmProviders[provider]; - const handleGenerate = async () => { + const handleGeneratePlan = async () => { if (!currentProvider.noApiKey && !apiKey.trim()) { setError("Please enter an API key"); return; } - if (!baseDirectoryPath.trim()) { setError("Please enter a base directory path"); return; } - // Create abort controller const controller = new AbortController(); setAbortController(controller); - setLoading(true); setError(null); - setStatus(`Generating script using ${currentProvider.name}...`); - - const fileSummary = buildFileSummary(files); - const filePatterns = analyzeFilePatterns(files); - const userContext = getUserContext(files); - const annotations = getFileAnnotations(files); - - // UPDATED: Improved prompt that uses trio files - const prompt = getConversionScriptPrompt( - baseDirectoryPath, - fileSummary, - filePatterns, - userContext, - annotations - ); + setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); try { - let response; - - if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // response = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { - // "Content-Type": "application/json", - // }, - // body: JSON.stringify({ - // model, - // messages: [ - // { - // role: "system", - // content: - // "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - // }, - // { role: "user", content: prompt }, - // ], - // stream: false, - // }), - // }); - response = await OllamaService.chat(model, [ - { - role: "system", - content: - "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ]); - } else if (currentProvider.isAnthropic) { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 4096, - messages: [{ role: "user", content: prompt }], - }), - }); - } else { - const headers: Record = { - "Content-Type": "application/json", - }; - - if (!currentProvider.noApiKey) { - headers["Authorization"] = `Bearer ${apiKey}`; - } + const { + planYaml, + subjectAnalysis: sa, + participantsTsv, + coverageWarnings, + } = await buildBidsPlan({ + evidenceBundle, + llmConfig: buildLLMConfig(), + signal: controller.signal, + onStatus: setStatus, + }); - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers, - body: JSON.stringify({ - model, - messages: [ - { - role: "system", - content: - "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ], - max_tokens: 4096, - temperature: 0.7, - }), + // Store subject analysis for ZIP packaging + setSubjectAnalysis(sa); + + // Dump final YAML string (planYaml is raw string from LLM, already cleaned) + setBidsPlan(planYaml); + + // Update participants.tsv in VFS with the full version from the plan stage + if (participantsTsv) { + const timestamp = new Date().toLocaleString(); + updateFiles((prev) => { + const withoutOld = prev.filter( + (f) => !(f.source === "ai" && f.name === "participants.tsv") + ); + return [ + ...withoutOld, + { + id: generateId(), + name: "participants.tsv", + type: "file" as const, + fileType: "meta", + content: participantsTsv, + contentType: "text", + isUserMeta: true, + parentId: null, + source: "ai" as const, + generatedAt: timestamp, + }, + ]; }); } - // const data = await response.json(); - const data = provider === "ollama" ? response : await response.json(); - - // if (!response.ok) { - // throw new Error(data.error?.message || "Failed to generate script"); - // } - if (!response.ok && provider !== "ollama") { - throw new Error(data.error?.message || "Failed to generate script"); + if (coverageWarnings.length > 0) { + setStatus( + `✓ BIDSPlan.yaml generated (${coverageWarnings.length} coverage warning(s) — check console)` + ); + } else { + setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); } - - // let script = ""; - // if (currentProvider.isAnthropic) { - // script = data.content[0].text; - // } else { - // script = data.choices[0].message.content; - // } - let script = currentProvider.isAnthropic - ? data.content[0].text - : data.choices[0].message.content; - - // Clean up markdown fences if AI included them anyway - script = script.replace(/^```python\n?/g, "").replace(/\n?```$/g, ""); - - setGeneratedScript(script); - setStatus(`✓ Script generated using ${currentProvider.name}`); } catch (err: any) { if (err.name === "AbortError") { setStatus("❌ Generation cancelled"); } else { - setError(err.message || "Failed to generate script"); - setStatus("❌ Error generating script"); + setError(err.message || "Failed to generate BIDSPlan"); + setStatus("❌ Error generating BIDSPlan"); } } finally { setLoading(false); - setAbortController(null); // Clear controller + setAbortController(null); } }; + // const handleGeneratePlan = async () => { + // if (!currentProvider.noApiKey && !apiKey.trim()) { + // setError("Please enter an API key"); + // return; + // } + // if (!baseDirectoryPath.trim()) { + // setError("Please enter a base directory path"); + // return; + // } - const handleGeneratePlan = async () => { - if (!currentProvider.noApiKey && !apiKey.trim()) { - setError("Please enter an API key"); - return; - } - if (!baseDirectoryPath.trim()) { - setError("Please enter a base directory path"); - return; - } - - const controller = new AbortController(); - setAbortController(controller); - setLoading(true); - setError(null); - setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); - - // ── Compute subject analysis (mirrors planner.py Step 1) - const allFiles = evidenceBundle?.all_files || []; - const userNSubjects = evidenceBundle?.user_hints?.n_subjects; - const dominantPrefixes = - evidenceBundle?.filename_analysis?.python_statistics?.dominant_prefixes; - - const computedSubjectAnalysis = extractSubjectAnalysis( - allFiles, - userNSubjects, - dominantPrefixes - ); - - setSubjectAnalysis(computedSubjectAnalysis); - - const fileSummary = buildFileSummary(files); - const filePatterns = analyzeFilePatterns(files); - const userContext = getUserContext(files); - // const subjectInfo = extractSubjectsFromFiles(files); - const subjectInfo = computedSubjectAnalysis; - const sampleFiles = - evidenceBundle?.samples - ?.slice(0, 10) - .map((s: any) => ` - ${s.relpath}`) - .join("\n") || ""; - - const prompt = getBIDSPlanPrompt( - fileSummary, - filePatterns, - userContext, - { - subjects: Object.entries( - computedSubjectAnalysis.id_mapping.id_mapping - ).map(([originalId, bidsId]) => ({ originalId, bidsId })), - strategy: computedSubjectAnalysis.id_mapping.strategy_used, - }, - evidenceBundle?.counts_by_ext || {}, - sampleFiles, - evidenceBundle - ); - - try { - let response; - - if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // response = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [ - // { - // role: "system", - // content: - // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - // }, - // { role: "user", content: prompt }, - // ], - // stream: false, - // }), - // }); - response = await OllamaService.chat(model, [ - { - role: "system", - content: - "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ]); - } else if (currentProvider.isAnthropic) { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: prompt }], - }), - }); - } else { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [ - { - role: "system", - content: - "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ], - max_tokens: 2048, - temperature: 0.15, - }), - }); - } - - // const data = await response.json(); + // const controller = new AbortController(); + // setAbortController(controller); + // setLoading(true); + // setError(null); + // setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); + + // // ── Compute subject analysis (mirrors planner.py Step 1) + // const allFiles = evidenceBundle?.all_files || []; + // const userNSubjects = evidenceBundle?.user_hints?.n_subjects; + // const dominantPrefixes = + // evidenceBundle?.filename_analysis?.python_statistics?.dominant_prefixes; + + // const computedSubjectAnalysis = extractSubjectAnalysis( + // allFiles, + // userNSubjects, + // dominantPrefixes + // ); - // if (!response.ok) { - // throw new Error(data.error?.message || "Failed to generate BIDSPlan"); - // } - const data = provider === "ollama" ? response : await response.json(); - if (!response.ok && provider !== "ollama") { - throw new Error(data.error?.message || "Failed to generate BIDSPlan"); - } + // setSubjectAnalysis(computedSubjectAnalysis); + + // const fileSummary = buildFileSummary(files); + // const filePatterns = analyzeFilePatterns(files); + // const userContext = getUserContext(files); + // // const subjectInfo = extractSubjectsFromFiles(files); + // const subjectInfo = computedSubjectAnalysis; + // const sampleFiles = + // evidenceBundle?.samples + // ?.slice(0, 10) + // .map((s: any) => ` - ${s.relpath}`) + // .join("\n") || ""; + + // const prompt = getBIDSPlanPrompt( + // fileSummary, + // filePatterns, + // userContext, + // { + // subjects: Object.entries( + // computedSubjectAnalysis.id_mapping.id_mapping + // ).map(([originalId, bidsId]) => ({ originalId, bidsId })), + // strategy: computedSubjectAnalysis.id_mapping.strategy_used, + // }, + // evidenceBundle?.counts_by_ext || {}, + // sampleFiles, + // evidenceBundle + // ); - let plan = currentProvider.isAnthropic - ? data.content[0].text - : data.choices[0].message.content; + // try { + // let response; + + // if (provider === "ollama") { + + // response = await OllamaService.chat(model, [ + // { + // role: "system", + // content: + // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", + // }, + // { role: "user", content: prompt }, + // ]); + // } else if (currentProvider.isAnthropic) { + // response = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: prompt }], + // }), + // }); + // } else { + // response = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [ + // { + // role: "system", + // content: + // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", + // }, + // { role: "user", content: prompt }, + // ], + // max_tokens: 2048, + // temperature: 0.15, + // }), + // }); + // } - // Clean up markdown fences if present - plan = plan - .replace(/^```yaml\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); + // const data = provider === "ollama" ? response : await response.json(); + // if (!response.ok && provider !== "ollama") { + // throw new Error(data.error?.message || "Failed to generate BIDSPlan"); + // } - setBidsPlan(plan); - setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); - } catch (err: any) { - if (err.name === "AbortError") { - setStatus("❌ Generation cancelled"); - } else { - setError(err.message || "Failed to generate BIDSPlan"); - setStatus("❌ Error generating BIDSPlan"); - } - } finally { - setLoading(false); - setAbortController(null); - } - }; + // let plan = currentProvider.isAnthropic + // ? data.content[0].text + // : data.choices[0].message.content; + + // // Clean up markdown fences if present + // plan = plan + // .replace(/^```yaml\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + + // setBidsPlan(plan); + // setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); + // } catch (err: any) { + // if (err.name === "AbortError") { + // setStatus("❌ Generation cancelled"); + // } else { + // setError(err.message || "Failed to generate BIDSPlan"); + // setStatus("❌ Error generating BIDSPlan"); + // } + // } finally { + // setLoading(false); + // setAbortController(null); + // } + // }; const handleDownloadPlan = () => { const blob = new Blob([bidsPlan], { type: "text/yaml" }); @@ -1518,7 +1499,7 @@ const LLMPanel: React.FC = ({ )} - {/* = ({ size="small" multiline rows={2} - /> */} + sx={{ mb: 1 }} + /> ), + ageRangeSlider: AgeRangeSliderField, }; // determine the results are subject-level or dataset-level @@ -406,6 +462,7 @@ const SearchPage: React.FC = () => { uiSchema={uiSchema} fields={customFields} widgets={customWidgets} + formContext={{ formData, setFormData }} /> ); @@ -847,6 +904,7 @@ const SearchPage: React.FC = () => { {...item} parsedJson={parsedJson} onChipClick={handleChipClick} + age={parsedJson?.key?.[0]} /> ); } catch (e) { diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 7b54428..d604b5f 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -52,10 +52,11 @@ export const generateUiSchema = ( "database", "keyword", "subject_filters_toggle", + "age_range_slider", // top of subject filters — range slider for age "modality", "type_name", // sits right after modality — its options depend on it "gender", - "age_min", + "age_min", // hidden via invisibleStyle; written by the slider above "age_max", "sess_min", "sess_max", @@ -101,16 +102,14 @@ export const generateUiSchema = ( : {} : hiddenStyle, - age_min: showSubjectFilters - ? formData["age_min"] - ? activeStyle - : {} - : hiddenStyle, - age_max: showSubjectFilters - ? formData["age_max"] - ? activeStyle - : {} + // Age range — slider lives inside the form via the AgeRangeSliderField + // stable component. age_min/age_max stay in the schema (so the backend + // gets them on submit) but their default numeric inputs are hidden. + age_range_slider: showSubjectFilters + ? { "ui:field": "ageRangeSlider" } : hiddenStyle, + age_min: invisibleStyle, + age_max: invisibleStyle, gender: showSubjectFilters ? formData["gender"] && formData["gender"] !== "any" diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index ead7855..bcba6d7 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -64,6 +64,10 @@ export const baseSchema: JSONSchema7 = { enum: ["male", "female", "unknown", "any"], default: "any", }, + age_range_slider: { + type: "null", + title: "Age range", + }, age_min: { title: "Minimum age", type: "number", From 88474cc4caf485da90a6a5ea366f69f208210ef3 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:05:54 -0400 Subject: [PATCH 24/61] feat(search): add placeholder hints to task/session/run keyword fields --- .../SearchPageFunctions/generateUiSchema.ts | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index d604b5f..e27d02c 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -151,9 +151,10 @@ export const generateUiSchema = ( : hiddenStyle, task_name: showSubjectFilters - ? formData["task_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. rest, motor", + ...(formData["task_name"] ? activeStyle : {}), + } : hiddenStyle, type_name: showSubjectFilters ? { @@ -167,14 +168,16 @@ export const generateUiSchema = ( } : hiddenStyle, session_name: showSubjectFilters - ? formData["session_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. 01, pre, baseline", + ...(formData["session_name"] ? activeStyle : {}), + } : hiddenStyle, run_name: showSubjectFilters - ? formData["run_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. 01, 02", + ...(formData["run_name"] ? activeStyle : {}), + } : hiddenStyle, "ui:submitButtonOptions": { From b9be29f248103c4c44e743c7274cb501d5c1b407 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:18:23 -0400 Subject: [PATCH 25/61] feat(search): pair min/max count fields on one row and tighten gaps(sessions, tasks, and runs) --- src/pages/SearchPage.tsx | 60 +++++++++++++++ .../SearchPageFunctions/generateUiSchema.ts | 76 +++++++++++-------- .../SearchPageFunctions/searchformSchema.ts | 3 + 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index e9ac04d..ca5d1c4 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -14,6 +14,8 @@ import { IconButton, Alert, Slider, + Stack, + TextField, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -101,6 +103,63 @@ const AgeRangeSliderField = (props: any) => { ); }; +// Pairs a "_min" + "_max" into a single row of two number inputs. +// Reads target field names + label from uiSchema's ui:options: +// { minKey: "sess_min", maxKey: "sess_max", label: "sessions" } +const CountRangePairField = (props: any) => { + const ctx = props?.registry?.formContext as + | { + formData: Record; + setFormData: React.Dispatch>>; + } + | undefined; + const opts = props?.uiSchema?.["ui:options"] || {}; + const minKey = opts.minKey as string; + const maxKey = opts.maxKey as string; + const label = (opts.label as string) || ""; + if (!ctx || !minKey || !maxKey) return null; + const { formData, setFormData } = ctx; + const minVal = formData[minKey] ?? ""; + const maxVal = formData[maxKey] ?? ""; + + const update = (key: string, raw: string) => { + setFormData((prev) => { + const next = { ...prev }; + if (raw === "" || raw === undefined) { + delete next[key]; + } else { + const n = Number(raw); + if (Number.isNaN(n)) delete next[key]; + else next[key] = n; + } + return next; + }); + }; + + return ( + + update(minKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + update(maxKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + + ); +}; + const matchesKeyword = (item: RegistryItem, keyword: string) => { if (!keyword) return false; const needle = keyword.toLowerCase(); @@ -308,6 +367,7 @@ const SearchPage: React.FC = () => { ), ageRangeSlider: AgeRangeSliderField, + countRangePair: CountRangePairField, }; // determine the results are subject-level or dataset-level diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index e27d02c..872accb 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -16,13 +16,11 @@ export const generateUiSchema = ( }, }; - // hide subject-level filter + // Fully remove a field from the rendered DOM (keeps its value in formData). + // Using ui:widget: "hidden" produces just an , so no + // empty Grid row + margin is left behind — fixes the big gap between rows. const invisibleStyle = { - "ui:options": { - style: { - display: "none", - }, - }, + "ui:widget": "hidden", }; const hiddenStyle = { @@ -58,10 +56,13 @@ export const generateUiSchema = ( "gender", "age_min", // hidden via invisibleStyle; written by the slider above "age_max", + "sess_count_range", // sessions min/max on one row "sess_min", "sess_max", + "task_count_range", // tasks min/max on one row "task_min", "task_max", + "run_count_range", // runs min/max on one row "run_min", "run_max", "task_name", @@ -117,38 +118,47 @@ export const generateUiSchema = ( : {} : hiddenStyle, - sess_min: showSubjectFilters - ? formData["sess_min"] - ? activeStyle - : {} - : hiddenStyle, - sess_max: showSubjectFilters - ? formData["sess_max"] - ? activeStyle - : {} + // Session / task / run min+max pairs are rendered by a single + // CountRangePairField each. The raw integer inputs are hidden but stay in + // formData so the backend still receives them on submit. + sess_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "sess_min", + maxKey: "sess_max", + label: "sessions", + }, + } : hiddenStyle, + sess_min: invisibleStyle, + sess_max: invisibleStyle, - task_min: showSubjectFilters - ? formData["task_min"] - ? activeStyle - : {} - : hiddenStyle, - task_max: showSubjectFilters - ? formData["task_max"] - ? activeStyle - : {} + task_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "task_min", + maxKey: "task_max", + label: "tasks", + }, + } : hiddenStyle, + task_min: invisibleStyle, + task_max: invisibleStyle, - run_min: showSubjectFilters - ? formData["run_min"] - ? activeStyle - : {} - : hiddenStyle, - run_max: showSubjectFilters - ? formData["run_max"] - ? activeStyle - : {} + run_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "run_min", + maxKey: "run_max", + label: "runs", + }, + } : hiddenStyle, + run_min: invisibleStyle, + run_max: invisibleStyle, task_name: showSubjectFilters ? { diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index bcba6d7..34abc4a 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -81,6 +81,7 @@ export const baseSchema: JSONSchema7 = { maximum: 1000, }, + sess_count_range: { type: "null", title: "Sessions" }, sess_min: { title: "Minimum session count", type: "integer", @@ -93,6 +94,7 @@ export const baseSchema: JSONSchema7 = { minimum: 0, maximum: 1000, }, + task_count_range: { type: "null", title: "Tasks" }, task_min: { title: "Minimum task count", type: "integer", @@ -105,6 +107,7 @@ export const baseSchema: JSONSchema7 = { minimum: 0, maximum: 1000, }, + run_count_range: { type: "null", title: "Runs" }, run_min: { title: "Minimum runs", type: "integer", From df73e1cbc0e2f61fe1c41c006a6dd08088d1871c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:31:47 -0400 Subject: [PATCH 26/61] feat(search): show run count in subject card --- src/components/SearchPage/SubjectCard.tsx | 9 ++++++++- src/utils/SearchPageFunctions/searchformSchema.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/SearchPage/SubjectCard.tsx b/src/components/SearchPage/SubjectCard.tsx index c66ebce..f805b48 100644 --- a/src/components/SearchPage/SubjectCard.tsx +++ b/src/components/SearchPage/SubjectCard.tsx @@ -16,6 +16,7 @@ interface SubjectCardProps { modalities?: string[]; tasks?: string[]; sessions?: string[]; + runs?: string[]; types?: string[]; }; }; @@ -32,7 +33,8 @@ const SubjectCard: React.FC = ({ index, onChipClick, }) => { - const { modalities, tasks, sessions, types } = parsedJson.value; + const { modalities, tasks, sessions, runs, types } = parsedJson.value; + const runCount = Array.isArray(runs) ? runs.length : 0; const subjectLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; const formattedSubj = /^sub-/i.test(subj) ? subj : `sub-${String(subj)}`; @@ -229,6 +231,11 @@ const SubjectCard: React.FC = ({ {sessions?.length === 0 ? 1 : sessions?.length} + + + Runs: {runCount} + + diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index 34abc4a..4cb4ef6 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -61,7 +61,7 @@ export const baseSchema: JSONSchema7 = { gender: { title: "Subject gender", type: "string", - enum: ["male", "female", "unknown", "any"], + enum: ["male", "female", "any"], default: "any", }, age_range_slider: { From 44635bd8a4e67c909306c43d68a0c8743bf24d2c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 10:55:48 -0400 Subject: [PATCH 27/61] feat(backend): add file type filter to search using iolinks table --- backend/src/controllers/couchdb.controller.js | 36 +++++++++++++++++++ backend/src/routes/dbs.routes.js | 6 ++++ 2 files changed, 42 insertions(+) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index f6c54c5..edb9957 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -227,6 +227,20 @@ const searchAllDatabases = async (req, res) => { repl.keyword = String(f.keyword); } + // File-type filter — array of extensions like [".jdb", ".snirf"]. + // Dataset-level: include rows whose (dbname, dsname) has at least one + // iolinks file with a matching view (extension). Per-subject filtering + // isn't possible here because iolinks.subj stores file size, not subj id. + if (Array.isArray(f.file_type) && f.file_type.length > 0) { + where.push(`EXISTS ( + SELECT 1 FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view = ANY(:fileTypes) + )`); + repl.fileTypes = f.file_type.map((t) => String(t)); + } + const limit = Math.min(parseInt(f.limit) || 100, 1000); const offset = parseInt(f.skip) || 0; repl.limit = limit; @@ -392,6 +406,27 @@ const getDatasetMeta = async (req, res) => { // } +// distinct file extensions present in iolinks across all synced DBs. +// Drives the multi-select "File types" filter on the search page. +const getFileTypes = async (req, res) => { + try { + const rows = await sequelize.query( + `SELECT DISTINCT view AS type + FROM iolinks + WHERE view IS NOT NULL AND view <> '' + ORDER BY view`, + { type: sequelize.QueryTypes.SELECT } + ); + res.status(200).json(rows.map((r) => r.type)); + } catch (error) { + console.error("Error fetching file types:", error.message); + res.status(500).json({ + message: "Error fetching file types", + error: error.message, + }); + } +}; + module.exports = { getDbList, getDbStats, @@ -400,4 +435,5 @@ module.exports = { searchAllDatabases, getDatasetDetail, getDatasetMeta, + getFileTypes, }; diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index c8a57fa..621b32f 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -6,6 +6,7 @@ const { getDbInfo, getDbDatasets, searchAllDatabases, + getFileTypes, // searchDatabase, } = require("../controllers/couchdb.controller"); @@ -15,6 +16,11 @@ const router = express.Router(); router.get("/", getDbList); router.get("/stats", getDbStats); +// distinct file extensions across all iolinks rows (drives the file-type +// filter on the search page). Must come BEFORE the /:dbName route, otherwise +// Express treats "file-types" as a dbName. +router.get("/file-types", getFileTypes); + // cross-database search router.post("/search", searchAllDatabases); From 1b0aa6576ecddb1c225adb3d8b0c543ea8069bb9 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 11:37:12 -0400 Subject: [PATCH 28/61] feat(search): add multi-select File types filter (dataset-level) --- backend/src/controllers/couchdb.controller.js | 4 +- .../widgets/FileTypeAutocompleteWidget.tsx | 40 +++++++++++++++++++ src/pages/SearchPage.tsx | 27 +++++++++++-- src/redux/neurojson/neurojson.action.ts | 13 ++++++ src/redux/neurojson/neurojson.slice.ts | 13 ++++++ .../neurojson/types/neurojson.interface.ts | 1 + src/services/neurojson.service.ts | 7 ++++ .../SearchPageFunctions/generateUiSchema.ts | 18 ++++++++- .../SearchPageFunctions/searchformSchema.ts | 6 +++ 9 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index edb9957..928a61b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -231,12 +231,14 @@ const searchAllDatabases = async (req, res) => { // Dataset-level: include rows whose (dbname, dsname) has at least one // iolinks file with a matching view (extension). Per-subject filtering // isn't possible here because iolinks.subj stores file size, not subj id. + // Use IN (:array) — Sequelize replacements expand arrays as 'a','b','c', + // which fits IN(...) but NOT ANY(...). if (Array.isArray(f.file_type) && f.file_type.length > 0) { where.push(`EXISTS ( SELECT 1 FROM iolinks l WHERE l.dbname = ioviews.dbname AND l.dsname = ioviews.dsname - AND l.view = ANY(:fileTypes) + AND l.view IN (:fileTypes) )`); repl.fileTypes = f.file_type.map((t) => String(t)); } diff --git a/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx new file mode 100644 index 0000000..eb941ad --- /dev/null +++ b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx @@ -0,0 +1,40 @@ +import { Autocomplete, Chip, TextField } from "@mui/material"; +import { WidgetProps } from "@rjsf/utils"; + +// Multi-select combobox for file extensions (e.g. ".jdb", ".snirf"). +// Options come from uiSchema's ui:options.fileTypes, fetched once by the +// parent SearchPage from /api/v1/dbs/file-types. +export const FileTypeAutocompleteWidget = (props: WidgetProps) => { + const { value, onChange, options, label } = props; + const fileTypes = (options.fileTypes as string[]) || []; + const current: string[] = Array.isArray(value) ? value : []; + + return ( + onChange(v as string[])} + renderTags={(items, getTagProps) => + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + + )} + /> + ); +}; diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ca5d1c4..e8e8da0 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -25,6 +25,7 @@ import ClickTooltip from "components/SearchPage/ClickTooltip"; import DatabaseCard from "components/SearchPage/DatabaseCard"; import DatasetCard from "components/SearchPage/DatasetCard"; import SubjectCard from "components/SearchPage/SubjectCard"; +import { FileTypeAutocompleteWidget } from "components/SearchPage/widgets/FileTypeAutocompleteWidget"; import { TypeAutocompleteWidget } from "components/SearchPage/widgets/TypeAutocompleteWidget"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; @@ -33,6 +34,7 @@ import pako from "pako"; import React from "react"; import { useState, useEffect, useMemo } from "react"; import { + fetchFileTypes, fetchMetadataSearchResults, fetchRegistry, } from "redux/neurojson/neurojson.action"; @@ -186,6 +188,9 @@ const SearchPage: React.FC = () => { const registry = useAppSelector( (state: RootState) => state.neurojson.registry ); + const fileTypes = useAppSelector( + (state: RootState) => state.neurojson.fileTypes + ); const loading = useAppSelector((state: RootState) => state.neurojson.loading); const [formData, setFormData] = useState>({}); @@ -323,12 +328,21 @@ const SearchPage: React.FC = () => { // form UI const uiSchema = useMemo( - () => generateUiSchema(formData, showSubjectFilters, showDatasetFilters), - [formData, showSubjectFilters, showDatasetFilters] + () => + generateUiSchema( + formData, + showSubjectFilters, + showDatasetFilters, + fileTypes || [] + ), + [formData, showSubjectFilters, showDatasetFilters, fileTypes] ); - // Custom RJSF widgets — combobox for the "Data type keywords" field. - const customWidgets = { typeAutocomplete: TypeAutocompleteWidget }; + // Custom RJSF widgets — comboboxes for the Data type and File types fields. + const customWidgets = { + typeAutocomplete: TypeAutocompleteWidget, + fileTypeAutocomplete: FileTypeAutocompleteWidget, + }; // Create the "Subject-level Filters" button as a custom field const customFields = { @@ -387,6 +401,11 @@ const SearchPage: React.FC = () => { dispatch(fetchRegistry()); }, [dispatch]); + // get the distinct file extensions for the "File types" multi-select. + useEffect(() => { + dispatch(fetchFileTypes()); + }, [dispatch]); + // dynamically add database enum to schema const schema = useMemo(() => { const dbList = registry?.length diff --git a/src/redux/neurojson/neurojson.action.ts b/src/redux/neurojson/neurojson.action.ts index 35cf6c1..f8f08c2 100644 --- a/src/redux/neurojson/neurojson.action.ts +++ b/src/redux/neurojson/neurojson.action.ts @@ -109,6 +109,19 @@ export const fetchMetadataSearchResults = createAsyncThunk( } ); +// distinct iolinks file extensions — populates the "File types" multi-select +export const fetchFileTypes = createAsyncThunk( + "neurojson/fetchFileTypes", + async (_, { rejectWithValue }) => { + try { + const data = await NeurojsonService.getFileTypes(); + return data; + } catch (error: any) { + return rejectWithValue("Failed to fetch file types"); + } + } +); + // fetch data for metadata panel in dataset detail page export const fetchDbInfoByDatasetId = createAsyncThunk( "neurojson/fetchDbInfoByDatasetId", diff --git a/src/redux/neurojson/neurojson.slice.ts b/src/redux/neurojson/neurojson.slice.ts index 90722c7..cfafc6a 100644 --- a/src/redux/neurojson/neurojson.slice.ts +++ b/src/redux/neurojson/neurojson.slice.ts @@ -7,6 +7,7 @@ import { fetchDbStats, fetchMetadataSearchResults, fetchDbInfoByDatasetId, + fetchFileTypes, } from "./neurojson.action"; import { DBDatafields, INeuroJsonState } from "./types/neurojson.interface"; import { createSlice, PayloadAction } from "@reduxjs/toolkit"; @@ -26,6 +27,7 @@ const initialState: INeuroJsonState = { dbStats: null, searchResults: null, datasetViewInfo: null, + fileTypes: null, }; const neurojsonSlice = createSlice({ @@ -155,6 +157,17 @@ const neurojsonSlice = createSlice({ state.loading = false; state.error = action.payload as string; }) + // fetchFileTypes runs once on mount; no pending case so it doesn't + // clobber the shared `loading` spinner used by the search button. + .addCase( + fetchFileTypes.fulfilled, + (state, action: PayloadAction) => { + state.fileTypes = action.payload; + } + ) + .addCase(fetchFileTypes.rejected, (state, action) => { + state.error = action.payload as string; + }) .addCase(fetchDbInfoByDatasetId.pending, (state) => { state.loading = true; state.error = null; diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index 365566d..01c8273 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -13,6 +13,7 @@ export interface INeuroJsonState { dbStats: DbStatsItem[] | null; // for dbStats on landing page searchResults: any[] | { status: string; msg: string } | null; datasetViewInfo: any | null; + fileTypes: string[] | null; } export interface DBParticulars { diff --git a/src/services/neurojson.service.ts b/src/services/neurojson.service.ts index 008e960..ed1f70d 100644 --- a/src/services/neurojson.service.ts +++ b/src/services/neurojson.service.ts @@ -150,6 +150,13 @@ export const NeurojsonService = { return response.data; }, + // GET /api/v1/dbs/file-types → distinct iolinks.view values + // Drives the multi-select "File types" filter on the search page. + getFileTypes: async (): Promise => { + const response = await api.get(`/dbs/file-types`); + return response.data; + }, + // getDbInfoByDatasetId: async (dbName: string, dsId: string): Promise => { // const response = await api.get( // `${baseURL}/${dbName}/_design/qq/_view/dbinfo`, diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 872accb..152a056 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -6,7 +6,8 @@ import { getTypeSuggestions } from "./typesByModality"; export const generateUiSchema = ( formData: Record, showSubjectFilters: boolean, - showDatasetFilters: boolean + showDatasetFilters: boolean, + fileTypeOptions: string[] = [] ) => { const activeStyle = { "ui:options": { @@ -49,6 +50,7 @@ export const generateUiSchema = ( "dataset_filters_toggle", // button first "database", "keyword", + "file_type", // dataset-level: filters by file extensions in iolinks "subject_filters_toggle", "age_range_slider", // top of subject filters — range slider for age "modality", @@ -90,6 +92,20 @@ export const generateUiSchema = ( // dataset: formData["dataset"] ? activeStyle : {}, // limit: formData["limit"] ? activeStyle : {}, // skip: formData["skip"] ? activeStyle : {}, + // File-type filter — dataset-level. Multi-select of file extensions + // present in iolinks (fetched dynamically via /api/v1/dbs/file-types). + file_type: showDatasetFilters + ? { + "ui:widget": "fileTypeAutocomplete", + "ui:options": { + fileTypes: fileTypeOptions, + ...(Array.isArray(formData["file_type"]) && + formData["file_type"].length > 0 + ? { style: { backgroundColor: Colors.lightBlue } } + : {}), + }, + } + : datasetHiddenStyle, limit: invisibleStyle, skip: invisibleStyle, diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index 4cb4ef6..e122725 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -128,6 +128,12 @@ export const baseSchema: JSONSchema7 = { title: "Data type", type: "string", }, + file_type: { + title: "File types", + type: "array", + items: { type: "string" }, + uniqueItems: true, + }, session_name: { title: "Session keywords", type: "string", From 9f826d4c4e959b9553c191ea8a52afd1585f4ea1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 12:12:11 -0400 Subject: [PATCH 29/61] fix(search): age slider per-handle clearing + non-BIDS file-type warning --- src/pages/SearchPage.tsx | 64 ++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index e8e8da0..260931c 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -81,16 +81,15 @@ const AgeRangeSliderField = (props: any) => { onChange={(_, v) => { const [newLo, newHi] = v as number[]; setFormData((prev) => { - const atFull = - newLo === AGE_MIN_BOUND && newHi === AGE_MAX_BOUND; const next = { ...prev }; - if (atFull) { - delete next.age_min; - delete next.age_max; - } else { - next.age_min = newLo; - next.age_max = newHi; - } + // Each handle is its own filter. A handle at the bound means + // "no constraint on that side", so we leave it out of formData + // (otherwise age_min=0 silently excludes unknown-age subjects + // whose stored key is "000-1", lexicographically below "00000"). + if (newLo === AGE_MIN_BOUND) delete next.age_min; + else next.age_min = newLo; + if (newHi === AGE_MAX_BOUND) delete next.age_max; + else next.age_max = newHi; return next; }); }} @@ -560,6 +559,35 @@ const SearchPage: React.FC = () => { !loading && // !hasDbMatches && (!hasDatasetMatches || backendEmpty); + + // Tailored empty-state message: when the user combined a file_type filter + // with any subject-level filter and got nothing back, it's almost certainly + // because the file extension lives in non-BIDS datasets (which have no + // subject rows in ioviews). The generic "adjust filters" message hides this. + const SUBJECT_FILTER_KEYS = [ + "age_min", + "age_max", + "gender", + "task_min", + "task_max", + "task_name", + "run_min", + "run_max", + "run_name", + "sess_min", + "sess_max", + "session_name", + "type_name", + "modality", + "subject", + ]; + const isAppliedFilter = (v: any) => + v !== "" && v !== "any" && v !== undefined && v !== null; + const showFileTypeNonBidsHint = + showNoResults && + Array.isArray(appliedFilters.file_type) && + appliedFilters.file_type.length > 0 && + SUBJECT_FILTER_KEYS.some((k) => isAppliedFilter(appliedFilters[k])); return ( { Search Results - - No datasets or subjects found. Please adjust the - filters and try again. - + {showFileTypeNonBidsHint ? ( + + No matching subjects found. The selected file type + may only exist in non-BIDS datasets (e.g. mesh or + atlas libraries), which have no subject-level + records. Try removing subject-level filters + (modality, age, gender, etc.) and search again. + + ) : ( + + No datasets or subjects found. Please adjust the + filters and try again. + + )} )} From 38fc1cbb83131613137d0386f55a4d6b2e993772 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 14:26:36 -0400 Subject: [PATCH 30/61] fix(search): highlight each word of multi-word keyword independently --- backend/src/controllers/couchdb.controller.js | 13 +++- src/components/SearchPage/DatasetCard.tsx | 75 +++++++++++++------ 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 928a61b..df1532b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -221,10 +221,19 @@ const searchAllDatabases = async (req, res) => { repl.subj = String(f.subject); } - // Keyword full-text search + // Keyword search — match anywhere relevant. + // tsquery covers stemmed tokens inside the JSON content (name, readme, + // info, modality, subj). ILIKE on dbname/dsname adds substring matching + // so "fnirs" finds "bfnirs", "openfnirs", and any dataset id containing it. + // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { - where.push(`search_vector @@ websearch_to_tsquery('english', :keyword)`); + where.push(`( + search_vector @@ websearch_to_tsquery('english', :keyword) + OR dbname ILIKE :keywordLike + OR dsname ILIKE :keywordLike + )`); repl.keyword = String(f.keyword); + repl.keywordLike = `%${String(f.keyword)}%`; } // File-type filter — array of extensions like [".jdb", ".snirf"]. diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 42f6646..5790ee5 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -35,11 +35,21 @@ const normalize = (s: string) => ?.replace(/[\u201C\u201D\u2033]/g, '"') ?? // curly → straight ""; +// Multi-word keyword support: backend tsquery treats "head brain" as AND of +// independent tokens. Highlighting should match the same logic — split on +// whitespace and treat each word independently. +const splitKeyword = (kw?: string): string[] => { + if (!kw) return []; + return normalize(kw).trim().split(/\s+/).filter(Boolean); +}; + +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const containsKeyword = (text?: string, kw?: string) => { if (!text || !kw) return false; const t = normalize(text).toLowerCase(); - const k = normalize(kw).toLowerCase(); - return t.includes(k); + const words = splitKeyword(kw.toLowerCase()); + return words.some((w) => t.includes(w)); }; /** Find a short snippet in secondary fields if not already visible */ @@ -62,24 +72,41 @@ function findMatchSnippet( ["ReferencesAndLinks", (v) => v?.info?.ReferencesAndLinks], ]; - const k = normalize(kw).toLowerCase(); + const words = splitKeyword(kw.toLowerCase()); + if (words.length === 0) return null; for (const [label, getter] of CANDIDATE_FIELDS) { const raw = getter(v); // v = parsedJson.value if (!raw) continue; const text = normalize(String(raw)); - const i = text.toLowerCase().indexOf(k); // k is the lowercase version of keyword - if (i >= 0) { - const start = Math.max(0, i - 40); - const end = Math.min(text.length, i + k.length + 40); - const before = text.slice(start, i); - const hit = text.slice(i, i + k.length); - const after = text.slice(i + k.length, end); - const html = `${ - start > 0 ? "…" : "" - }${before}${hit}${after}${end < text.length ? "…" : ""}`; - return { label, html }; + const lower = text.toLowerCase(); + + // Find the earliest occurrence of ANY matching word — that's the snippet anchor. + let anchor = -1; + let anchorLen = 0; + for (const w of words) { + const i = lower.indexOf(w); + if (i >= 0 && (anchor < 0 || i < anchor)) { + anchor = i; + anchorLen = w.length; + } } + if (anchor < 0) continue; + + const start = Math.max(0, anchor - 40); + const end = Math.min(text.length, anchor + anchorLen + 40); + const slice = text.slice(start, end); + + // Highlight every matching word inside the snippet, not just the first. + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + const highlighted = slice.replace(regex, "$1"); + const html = `${start > 0 ? "…" : ""}${highlighted}${ + end < text.length ? "…" : "" + }`; + return { label, html }; } return null; } @@ -122,19 +149,25 @@ const DatasetCard: React.FC = ({ [parsedJson.value, keyword, visibleHasKeyword] ); - // keyword highlight functional component (only for visible fields) + // keyword highlight functional component (only for visible fields). + // Splits the keyword on whitespace and highlights each word independently + // so "head brain" highlights both words wherever they appear. const highlightKeyword = (text: string, keyword?: string) => { - if (!keyword || !text?.toLowerCase().includes(keyword.toLowerCase())) { - return text; - } - - const regex = new RegExp(`(${keyword})`, "gi"); // for case-insensitive and global + const words = splitKeyword(keyword); + if (words.length === 0 || !text) return text; + const lowerWordSet = new Set(words.map((w) => w.toLowerCase())); + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + if (!regex.test(text)) return text; + // Reset lastIndex because test() advances on /g regexes; safer to use split. const parts = text.split(regex); return ( <> {parts.map((part, i) => - part.toLowerCase() === keyword.toLowerCase() ? ( + lowerWordSet.has(part.toLowerCase()) ? ( Date: Tue, 12 May 2026 16:31:53 -0400 Subject: [PATCH 31/61] fix(search): keyword highlight reads from appliedFilters, not formData --- src/pages/SearchPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 260931c..ca63bed 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1002,7 +1002,7 @@ const SearchPage: React.FC = () => { dsname={item.dsname} parsedJson={parsedJson} onChipClick={handleChipClick} - keyword={formData.keyword} // for keyword highlight + keyword={appliedFilters.keyword} // highlight what was searched, not the live form /> ) : ( Date: Wed, 13 May 2026 15:26:18 -0400 Subject: [PATCH 32/61] feat: add file download endpoints for dataset search results --- backend/src/controllers/couchdb.controller.js | 76 ++++++++++++++++++- backend/src/routes/dbs.routes.js | 5 ++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index df1532b..db642da 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -257,6 +257,32 @@ const searchAllDatabases = async (req, res) => { repl.limit = limit; repl.offset = offset; + // When file_type filter is active, also return a sample of the actual + // matching iolinks rows (filename, url, path, suffix) per dataset, plus + // a total count. Frontend shows up to 20 as clickable filenames and a + // "Download manifest" button for the full list via a separate endpoint. + const matchingFilesActive = + Array.isArray(f.file_type) && f.file_type.length > 0; + const matchingFilesColumn = matchingFilesActive + ? `, + COALESCE(( + SELECT jsonb_agg(t.json) + FROM ( + SELECT l.json + FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes) + ORDER BY l.id + LIMIT 20 + ) t + ), '[]'::jsonb)::text AS matching_files, + (SELECT COUNT(*) FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes))::int AS matching_files_total` + : ""; + // dbinfo was stored flat ({name, subj, ...}); subjects was stored wrapped // ({key, value}). Frontend expects parsed.value.subj for datasets, so we // wrap dbinfo on the way out. @@ -268,7 +294,7 @@ const searchAllDatabases = async (req, res) => { CASE WHEN view = 'dbinfo' THEN jsonb_build_object('value', json)::text ELSE json::text - END AS json + END AS json${matchingFilesColumn} FROM ioviews WHERE ${where.join(" AND ")} ORDER BY dbname, dsname, subj @@ -417,6 +443,53 @@ const getDatasetMeta = async (req, res) => { // } +// Plain-text manifest of every matching iolinks URL for a dataset, served +// as a downloadable .txt. The user pipes it into wget/aria2c to fetch +// everything: `wget -i manifest.txt`. Avoids server-side zipping and gives +// resumable, parallel downloads. +const getDatasetFilesManifest = async (req, res) => { + try { + const { dbName, dsName } = req.params; + const rawExt = req.query.ext; + const exts = Array.isArray(rawExt) + ? rawExt + : typeof rawExt === "string" && rawExt.length > 0 + ? rawExt.split(",") + : []; + + if (exts.length === 0) { + res.status(400).send("ext query parameter required (e.g. ?ext=.jdb)"); + return; + } + + const rows = await sequelize.query( + `SELECT json->'value'->>'url' AS url + FROM iolinks + WHERE dbname = :dbname + AND dsname = :dsname + AND view IN (:exts) + ORDER BY id`, + { + replacements: { dbname: dbName, dsname: dsName, exts }, + type: sequelize.QueryTypes.SELECT, + } + ); + + const urls = rows.map((r) => r.url).filter(Boolean); + const filename = `${dbName}_${dsName}_${exts.join("_")}_manifest.txt`; + + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${filename}"` + ); + res.send(urls.join("\n") + "\n"); + } catch (error) { + console.error("Error generating manifest:", error.message); + res.status(500).send(`Error generating manifest: ${error.message}`); + } +}; + // distinct file extensions present in iolinks across all synced DBs. // Drives the multi-select "File types" filter on the search page. const getFileTypes = async (req, res) => { @@ -447,4 +520,5 @@ module.exports = { getDatasetDetail, getDatasetMeta, getFileTypes, + getDatasetFilesManifest, }; diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index 621b32f..45979ac 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -7,6 +7,7 @@ const { getDbDatasets, searchAllDatabases, getFileTypes, + getDatasetFilesManifest, // searchDatabase, } = require("../controllers/couchdb.controller"); @@ -24,6 +25,10 @@ router.get("/file-types", getFileTypes); // cross-database search router.post("/search", searchAllDatabases); +// downloadable manifest (plain text) of all iolinks URLs for a dataset +// filtered by extension(s). e.g. /dbs/bfnirs/Motion-Yucel2014-I/files/manifest?ext=.jdb +router.get("/:dbName/:dsName/files/manifest", getDatasetFilesManifest); + // Specific database routes router.get("/:dbName", getDbInfo); router.get("/:dbName/datasets", getDbDatasets); From 51960cf2274aacf95c4864da675809b74aadd925 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 13 May 2026 15:45:06 -0400 Subject: [PATCH 33/61] feat(search): show matching files in dataset card with selective and manifest download --- backend/src/controllers/couchdb.controller.js | 4 +- src/components/SearchPage/DatasetCard.tsx | 145 +++++++++++++++++- src/pages/SearchPage.tsx | 7 + 3 files changed, 153 insertions(+), 3 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index db642da..20a7cae 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -259,7 +259,7 @@ const searchAllDatabases = async (req, res) => { // When file_type filter is active, also return a sample of the actual // matching iolinks rows (filename, url, path, suffix) per dataset, plus - // a total count. Frontend shows up to 20 as clickable filenames and a + // a total count. Frontend shows up to 10 as clickable filenames and a // "Download manifest" button for the full list via a separate endpoint. const matchingFilesActive = Array.isArray(f.file_type) && f.file_type.length > 0; @@ -274,7 +274,7 @@ const searchAllDatabases = async (req, res) => { AND l.dsname = ioviews.dsname AND l.view IN (:fileTypes) ORDER BY l.id - LIMIT 20 + LIMIT 10 ) t ), '[]'::jsonb)::text AS matching_files, (SELECT COUNT(*) FROM iolinks l diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 5790ee5..da4e0c7 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -1,10 +1,31 @@ -import { Typography, Card, CardContent, Stack, Chip } from "@mui/material"; +import DownloadIcon from "@mui/icons-material/Download"; +import { + Typography, + Card, + CardContent, + Stack, + Chip, + Button, + Link as MuiLink, +} from "@mui/material"; +import { baseURL } from "services/instance"; import { Colors } from "design/theme"; import React from "react"; import { useMemo } from "react"; import { Link } from "react-router-dom"; import RoutesEnum from "types/routes.enum"; +interface MatchingFile { + key?: any; + value?: { + file?: string; + url?: string; + path?: string; + suffix?: string; + ref?: string; + }; +} + interface DatasetCardProps { dbname: string; dsname: string; @@ -26,6 +47,9 @@ interface DatasetCardProps { index: number; onChipClick: (key: string, value: string) => void; keyword?: string; // for keyword highlight + matchingFiles?: MatchingFile[]; // sample of iolinks rows matching file_type + matchingFilesTotal?: number; // total count across all matches + fileTypes?: string[]; // the active file_type filter, used to build manifest URL } /** ---------- utility helpers ---------- **/ @@ -119,10 +143,41 @@ const DatasetCard: React.FC = ({ index, onChipClick, keyword, + matchingFiles, + matchingFilesTotal, + fileTypes, }) => { const { name, readme, modality, subj, info } = parsedJson.value; const datasetLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; + // Manifest URL — backend serves a plain-text list of all matching URLs. + const manifestUrl = useMemo(() => { + if (!fileTypes || fileTypes.length === 0) return null; + const ext = fileTypes + .map((e) => encodeURIComponent(e)) + .join(","); + return `${baseURL}/dbs/${encodeURIComponent( + dbname + )}/${encodeURIComponent(dsname)}/files/manifest?ext=${ext}`; + }, [dbname, dsname, fileTypes]); + + // Extract a short "sub-XXX" tag from a BIDS path like + // "$.sub-019.ses-1.nirs.sub-019_ses-1_task-MA_run-01_nirs.snirf.SNIRFData..." + const subjectFromPath = (p?: string): string => { + if (!p) return ""; + const m = p.match(/sub-[^.]+/); + return m ? m[0] : ""; + }; + + // File size stored in key[1] of each iolinks row (bytes). Format for humans. + const formatBytes = (n?: number): string => { + if (typeof n !== "number" || !Number.isFinite(n) || n < 0) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; + }; + // prepare DOI URL const rawDOI = info?.DatasetDOI?.replace(/^doi:/, ""); const doiLink = rawDOI ? `https://doi.org/${rawDOI}` : null; @@ -338,6 +393,94 @@ const DatasetCard: React.FC = ({ )} + + {/* Matching files section — only shown when file_type filter is active */} + {Array.isArray(matchingFiles) && matchingFiles.length > 0 && ( + + + + Matching files + {typeof matchingFilesTotal === "number" && + ` (${ + matchingFiles.length < matchingFilesTotal + ? `${matchingFiles.length} of ${matchingFilesTotal}` + : matchingFilesTotal + })`} + + {manifestUrl && ( + + )} + + + {matchingFiles.slice(0, 10).map((f, i) => { + const v = f.value || {}; + const subjTag = subjectFromPath(v.path); + const sizeBytes = + Array.isArray(f.key) && typeof f.key[1] === "number" + ? f.key[1] + : undefined; + const sizeTag = formatBytes(sizeBytes); + const meta = [subjTag, sizeTag].filter(Boolean).join(" · "); + return ( +
  • + + {v.file || v.url} + + {meta && ( + + ({meta}) + + )} +
    • + ); + })} +     +     + )} diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ca63bed..397e63a 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1003,6 +1003,13 @@ const SearchPage: React.FC = () => { parsedJson={parsedJson} onChipClick={handleChipClick} keyword={appliedFilters.keyword} // highlight what was searched, not the live form + matchingFiles={ + item.matching_files + ? JSON.parse(item.matching_files) + : undefined + } + matchingFilesTotal={item.matching_files_total} + fileTypes={appliedFilters.file_type} /> ) : ( Date: Wed, 13 May 2026 16:14:45 -0400 Subject: [PATCH 34/61] fix(search): use plainto_tsquery + normalize ILIKE separators for keyword --- backend/src/controllers/couchdb.controller.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 20a7cae..cdcedd3 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -222,18 +222,23 @@ const searchAllDatabases = async (req, res) => { } // Keyword search — match anywhere relevant. - // tsquery covers stemmed tokens inside the JSON content (name, readme, - // info, modality, subj). ILIKE on dbname/dsname adds substring matching - // so "fnirs" finds "bfnirs", "openfnirs", and any dataset id containing it. + // plainto_tsquery treats input as plain words AND'd together; ignores + // operator chars like "-" and "OR" so dataset names with hyphens + // (e.g. "ABIDE - CMU_a") don't get parsed as NOT clauses. + // ILIKE on dbname/dsname adds substring matching so "fnirs" finds + // "bfnirs", "openfnirs", and any dataset id containing it. + // ILIKE pattern normalizes whitespace/hyphens to % wildcards so the + // user's "ABIDE - CMU_a" matches stored names like "abide_cmu_a" or + // "ABIDE_-_CMU_a" regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { where.push(`( - search_vector @@ websearch_to_tsquery('english', :keyword) + search_vector @@ plainto_tsquery('english', :keyword) OR dbname ILIKE :keywordLike OR dsname ILIKE :keywordLike )`); repl.keyword = String(f.keyword); - repl.keywordLike = `%${String(f.keyword)}%`; + repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; } // File-type filter — array of extensions like [".jdb", ".snirf"]. From bf1abe34995b9e28692165aa38b07718fdce90c6 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 13 May 2026 16:31:14 -0400 Subject: [PATCH 35/61] fix(search): match dataset display name via json->>'name' ILIKE --- backend/src/controllers/couchdb.controller.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index cdcedd3..88e96f1 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -227,15 +227,19 @@ const searchAllDatabases = async (req, res) => { // (e.g. "ABIDE - CMU_a") don't get parsed as NOT clauses. // ILIKE on dbname/dsname adds substring matching so "fnirs" finds // "bfnirs", "openfnirs", and any dataset id containing it. - // ILIKE pattern normalizes whitespace/hyphens to % wildcards so the - // user's "ABIDE - CMU_a" matches stored names like "abide_cmu_a" or - // "ABIDE_-_CMU_a" regardless of separator style. + // ILIKE on json->>'name' covers the human-readable name from + // dataset_description.json (e.g. "ABIDE - CMU_a"), which is where the + // user-visible dataset titles live — dsname column often stores just + // an opaque id like "CMU_a" without the prefix. + // ILIKE pattern normalizes whitespace/hyphens to % wildcards so + // "ABIDE - CMU_a" matches stored names regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { where.push(`( search_vector @@ plainto_tsquery('english', :keyword) OR dbname ILIKE :keywordLike OR dsname ILIKE :keywordLike + OR (json->>'name') ILIKE :keywordLike )`); repl.keyword = String(f.keyword); repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; From 6fce0633ef5b639c863302ff50c659386d0ce15b Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 14 May 2026 16:56:12 -0400 Subject: [PATCH 36/61] feat(search): add Mac/Linux and Windows script options to file download --- backend/src/controllers/couchdb.controller.js | 86 ++++++++- src/components/SearchPage/DatasetCard.tsx | 177 +++++++++++++++--- 2 files changed, 229 insertions(+), 34 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 88e96f1..c990fe7 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -452,14 +452,18 @@ const getDatasetMeta = async (req, res) => { // } -// Plain-text manifest of every matching iolinks URL for a dataset, served -// as a downloadable .txt. The user pipes it into wget/aria2c to fetch -// everything: `wget -i manifest.txt`. Avoids server-side zipping and gives -// resumable, parallel downloads. +// Downloadable list of every matching iolinks URL for a dataset. +// Three formats via ?format=: +// - txt (default) → plain URL list (use with `wget -i`) +// - sh → bash script with curl commands (Mac/Linux) +// - bat → Windows batch script with curl commands +// All three avoid server-side zipping — the user's machine pulls files +// directly from neurojson.org/io, so this Express server stays light. const getDatasetFilesManifest = async (req, res) => { try { const { dbName, dsName } = req.params; const rawExt = req.query.ext; + const format = String(req.query.format || "txt").toLowerCase(); const exts = Array.isArray(rawExt) ? rawExt : typeof rawExt === "string" && rawExt.length > 0 @@ -472,7 +476,8 @@ const getDatasetFilesManifest = async (req, res) => { } const rows = await sequelize.query( - `SELECT json->'value'->>'url' AS url + `SELECT json->'value'->>'url' AS url, + json->'value'->>'file' AS file FROM iolinks WHERE dbname = :dbname AND dsname = :dsname @@ -484,15 +489,78 @@ const getDatasetFilesManifest = async (req, res) => { } ); - const urls = rows.map((r) => r.url).filter(Boolean); - const filename = `${dbName}_${dsName}_${exts.join("_")}_manifest.txt`; + const files = rows.filter((r) => r.url); + const urls = files.map((r) => r.url); + const baseName = `${dbName}_${dsName}_${exts.join("_")}`; + const extLabel = exts.join(", "); + + // Strip any path separators or quote chars from the parsed filename + // before using it in shell commands — file names come from iolinks + // and are usually content hashes, but defensive belt-and-suspenders. + const safeName = (s) => + (s || "").replace(/["\\\/\r\n]/g, "").trim(); + + let body; + let contentType; + let filename; + + if (format === "sh") { + // Bash script — curl is preinstalled on macOS and most Linux distros. + // -L follows redirects, -C - resumes interrupted downloads, -o saves + // with our parsed filename (the URL is a CGI query — using -O would + // save files as literal `stat.cgi?...`). + body = + `#!/bin/bash\n` + + `# Downloads ${extLabel} files from ${dbName}/${dsName}\n` + + `# Usage: bash ${baseName}_download.sh\n` + + `set -e\n` + + `mkdir -p "neurojson_downloads"\n` + + `cd "neurojson_downloads" || exit 1\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + contentType = "application/x-sh; charset=utf-8"; + filename = `${baseName}_download.sh`; + } else if (format === "bat") { + // Windows batch — curl ships with Windows 10+. Uses CRLF line endings + // for proper rendering in CMD. /d on cd handles cross-drive paths. + body = + `@echo off\r\n` + + `REM Downloads ${extLabel} files from ${dbName}/${dsName}\r\n` + + `REM Usage: double-click or run ${baseName}_download.bat\r\n` + + `if not exist "neurojson_downloads" mkdir "neurojson_downloads"\r\n` + + `cd /d "neurojson_downloads"\r\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_download.bat`; + } else { + // Default: plain URL list, one per line (advanced users with wget). + body = urls.join("\n") + "\n"; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_manifest.txt`; + } - res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.setHeader("Content-Type", contentType); res.setHeader( "Content-Disposition", `attachment; filename="${filename}"` ); - res.send(urls.join("\n") + "\n"); + res.send(body); } catch (error) { console.error("Error generating manifest:", error.message); res.status(500).send(`Error generating manifest: ${error.message}`); diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index da4e0c7..4243119 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -1,4 +1,5 @@ import DownloadIcon from "@mui/icons-material/Download"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import { Typography, Card, @@ -7,11 +8,16 @@ import { Chip, Button, Link as MuiLink, + Menu, + MenuItem, + Box, + Snackbar, + Alert, } from "@mui/material"; import { baseURL } from "services/instance"; import { Colors } from "design/theme"; import React from "react"; -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { Link } from "react-router-dom"; import RoutesEnum from "types/routes.enum"; @@ -150,16 +156,44 @@ const DatasetCard: React.FC = ({ const { name, readme, modality, subj, info } = parsedJson.value; const datasetLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; - // Manifest URL — backend serves a plain-text list of all matching URLs. - const manifestUrl = useMemo(() => { + // Build manifest URL for any of the three formats. Backend serves + // text/plain for .txt, application/x-sh for .sh, text/plain for .bat — + // each with a Content-Disposition header so the browser saves them. + const buildManifestUrl = (format: "txt" | "sh" | "bat") => { if (!fileTypes || fileTypes.length === 0) return null; - const ext = fileTypes - .map((e) => encodeURIComponent(e)) - .join(","); + const ext = fileTypes.map((e) => encodeURIComponent(e)).join(","); return `${baseURL}/dbs/${encodeURIComponent( dbname - )}/${encodeURIComponent(dsname)}/files/manifest?ext=${ext}`; - }, [dbname, dsname, fileTypes]); + )}/${encodeURIComponent( + dsname + )}/files/manifest?ext=${ext}&format=${format}`; + }; + + const hasManifest = Array.isArray(fileTypes) && fileTypes.length > 0; + + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses it + // (no autoHideDuration) so researchers have time to read multi-step + // instructions. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); + const handleDownload = (format: "txt" | "sh" | "bat") => { + const url = buildManifestUrl(format); + setDownloadMenuEl(null); + if (!url) return; + // Programmatic anchor click triggers the browser's normal download flow + // without leaving the current page (window.location would navigate away). + const a = document.createElement("a"); + a.href = url; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setDownloadHint(format); + }; // Extract a short "sub-XXX" tag from a BIDS path like // "$.sub-019.ses-1.nirs.sub-019_ses-1_task-MA_run-01_nirs.snirf.SNIRFData..." @@ -421,23 +455,40 @@ const DatasetCard: React.FC = ({ : matchingFilesTotal })`} - {manifestUrl && ( - + {hasManifest && ( + <> + + setDownloadMenuEl(null)} + > + handleDownload("sh")}> + For Mac / Linux (.sh) + + handleDownload("bat")}> + For Windows (.bat) + + handleDownload("txt")}> + URL list (.txt, advanced) + + + )} @@ -483,6 +534,82 @@ const DatasetCard: React.FC = ({ )} + + {/* Post-download instructions. No auto-hide so users can read at their + * own pace; dismiss with the ✕ when finished. */} + setDownloadHint(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setDownloadHint(null)} + sx={{ maxWidth: 520 }} + > + {downloadHint === "sh" && ( + + + Downloaded the Mac / Linux script + + + To fetch your data files: + + +
  • Open Terminal
    • +
  • Go to the folder where the script was saved
    • +
  • + Run:{" "} + + bash <script-name>.sh + +
    • +     +     + )} + {downloadHint === "bat" && ( + + + Downloaded the Windows script + + + Open the folder where the script was saved and{" "} + double-click the .bat file. A command window + opens and the files download next to it. + + + )} + {downloadHint === "txt" && ( + + + Downloaded the URL list + + + In Terminal (Mac/Linux) or PowerShell (Windows), run:{" "} + + wget -i <file-name>.txt + + + + )} +     +     ); }; From 8be3fa1af653de11aedc926130f7a054a2d09bf2 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 15 May 2026 14:15:49 -0400 Subject: [PATCH 37/61] fix(dataset-detail): keep all downloaded files in one visible folder --- src/pages/UpdatedDatasetDetailPage.tsx | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/pages/UpdatedDatasetDetailPage.tsx b/src/pages/UpdatedDatasetDetailPage.tsx index f94a751..734c104 100644 --- a/src/pages/UpdatedDatasetDetailPage.tsx +++ b/src/pages/UpdatedDatasetDetailPage.tsx @@ -577,8 +577,17 @@ const UpdatedDatasetDetailPage: React.FC = () => { // }); // setJsonSize(blob.size); - // Construct download script dynamically - let script = `curl -L --create-dirs "https://neurojson.io:7777/${dbName}/${docId}" -o "${docId}.json"\n`; + // Construct download script dynamically — everything lands in a + // .// folder next to where the user runs the script, so the + // JSON and the data files stay together (was split between cwd and + // ~/.neurojson/io/... previously, hard to find). + let script = `#!/bin/bash\n`; + script += `# Downloads ${docId} from ${dbName}\n`; + script += `# Usage: bash ${docId}.sh\n`; + script += `set -e\n`; + script += `mkdir -p "${docId}"\n`; + script += `cd "${docId}" || exit 1\n`; + script += `curl -L -C - -o "${docId}.json" "https://neurojson.io:7777/${dbName}/${docId}"\n`; links.forEach((link) => { const url = link.url; @@ -594,10 +603,10 @@ const UpdatedDatasetDetailPage: React.FC = () => { })() : `file-${link.index}`; - const outputPath = `$HOME/.neurojson/io/${dbName}/${docId}/${filename}`; - - script += `curl -L --create-dirs "${url}" -o "${outputPath}"\n`; + script += `curl -L -C - -o "${filename}" "${url}"\n`; }); + + script += `echo "Done. Files saved to $(pwd)"\n`; setDownloadScript(script); // Calculate and set script size const scriptBlob = new Blob([script], { type: "text/plain" }); From e21c661760d3061f2b18540043c8952c60a259fa Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 15 May 2026 14:24:22 -0400 Subject: [PATCH 38/61] feat(dataset-detail): three script formats for download all files button --- src/pages/UpdatedDatasetDetailPage.tsx | 223 ++++++++++++++++++++----- 1 file changed, 183 insertions(+), 40 deletions(-) diff --git a/src/pages/UpdatedDatasetDetailPage.tsx b/src/pages/UpdatedDatasetDetailPage.tsx index 734c104..835a6b5 100644 --- a/src/pages/UpdatedDatasetDetailPage.tsx +++ b/src/pages/UpdatedDatasetDetailPage.tsx @@ -7,6 +7,7 @@ import ExpandLess from "@mui/icons-material/ExpandLess"; import ExpandMore from "@mui/icons-material/ExpandMore"; import HomeIcon from "@mui/icons-material/Home"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import { Box, Typography, @@ -17,6 +18,9 @@ import { Collapse, Tooltip, IconButton, + Menu, + MenuItem, + Snackbar, } from "@mui/material"; import DatasetActions from "components/DatasetDetailPage/DatasetAction"; import FileTree from "components/DatasetDetailPage/FileTree/FileTree"; @@ -267,8 +271,23 @@ const UpdatedDatasetDetailPage: React.FC = () => { const [externalLinks, setExternalLinks] = useState([]); const [internalLinks, setInternalLinks] = useState([]); const [isInternalExpanded, setIsInternalExpanded] = useState(true); - const [downloadScript, setDownloadScript] = useState(""); + // Three script formats generated client-side: bash (Mac/Linux), batch + // (Windows), and a plain URL list. Same files in all three; only the + // wrapper syntax differs. + const [downloadScripts, setDownloadScripts] = useState<{ + sh: string; + bat: string; + txt: string; + }>({ sh: "", bat: "", txt: "" }); const [downloadScriptSize, setDownloadScriptSize] = useState(0); + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); const [totalFileSize, setTotalFileSize] = useState(0); const [previewIsInternal, setPreviewIsInternal] = useState(false); const [isExternalExpanded, setIsExternalExpanded] = useState(true); @@ -577,40 +596,62 @@ const UpdatedDatasetDetailPage: React.FC = () => { // }); // setJsonSize(blob.size); - // Construct download script dynamically — everything lands in a - // .// folder next to where the user runs the script, so the - // JSON and the data files stay together (was split between cwd and - // ~/.neurojson/io/... previously, hard to find). - let script = `#!/bin/bash\n`; - script += `# Downloads ${docId} from ${dbName}\n`; - script += `# Usage: bash ${docId}.sh\n`; - script += `set -e\n`; - script += `mkdir -p "${docId}"\n`; - script += `cd "${docId}" || exit 1\n`; - script += `curl -L -C - -o "${docId}.json" "https://neurojson.io:7777/${dbName}/${docId}"\n`; - - links.forEach((link) => { - const url = link.url; - const match = url.match(/file=([^&]+)/); - - const filename = match - ? (() => { - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; // fallback if decode fails - } - })() - : `file-${link.index}`; - - script += `curl -L -C - -o "${filename}" "${url}"\n`; - }); - - script += `echo "Done. Files saved to $(pwd)"\n`; - setDownloadScript(script); - // Calculate and set script size - const scriptBlob = new Blob([script], { type: "text/plain" }); - setDownloadScriptSize(scriptBlob.size); + // Construct download scripts (three formats) dynamically — everything + // lands in a .// folder next to where the user runs the script. + // JSON metadata and data files stay together (was split between cwd + // and ~/.neurojson/io/... previously, hard to find). + const docUrl = `https://neurojson.io:7777/${dbName}/${docId}`; + type DlItem = { url: string; filename: string }; + const items: DlItem[] = [ + { url: docUrl, filename: `${docId}.json` }, + ...links.map((link) => { + const match = link.url.match(/file=([^&]+)/); + const filename = match + ? (() => { + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } + })() + : `file-${link.index}`; + return { url: link.url, filename }; + }), + ]; + + // Bash script (Mac/Linux) + const sh = + `#!/bin/bash\n` + + `# Downloads ${docId} from ${dbName}\n` + + `# Usage: bash ${docId}.sh\n` + + `set -e\n` + + `mkdir -p "${docId}"\n` + + `cd "${docId}" || exit 1\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + + // Batch script (Windows) — curl ships with Windows 10+. CRLF endings. + const bat = + `@echo off\r\n` + + `REM Downloads ${docId} from ${dbName}\r\n` + + `REM Usage: double-click or run ${docId}.bat\r\n` + + `if not exist "${docId}" mkdir "${docId}"\r\n` + + `cd /d "${docId}"\r\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + + // Plain URL list — for advanced users with wget. + const txt = items.map((it) => it.url).join("\n") + "\n"; + + setDownloadScripts({ sh, bat, txt }); + // Size shown on the button is the .sh script size (representative). + const shBlob = new Blob([sh], { type: "text/plain" }); + setDownloadScriptSize(shBlob.size); } }, [datasetDocument, docId]); @@ -636,14 +677,25 @@ const UpdatedDatasetDetailPage: React.FC = () => { document.body.removeChild(link); }; - const handleDownloadScript = () => { - const blob = new Blob([downloadScript], { type: "text/plain" }); + // Trigger download of the selected script format. Programmatic anchor + // click triggers the browser's normal download flow without navigating. + const handleDownloadScript = (format: "sh" | "bat" | "txt") => { + const content = downloadScripts[format]; + if (!content) return; + const mime = + format === "sh" ? "application/x-sh" : "text/plain"; + const filename = + format === "txt" ? `${docId}_manifest.txt` : `${docId}.${format}`; + const blob = new Blob([content], { type: `${mime}; charset=utf-8` }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = `${docId}.sh`; + link.download = filename; document.body.appendChild(link); link.click(); document.body.removeChild(link); + URL.revokeObjectURL(link.href); + setDownloadMenuEl(null); + setDownloadHint(format); }; const handlePreview = ( @@ -1127,20 +1179,35 @@ const UpdatedDatasetDetailPage: React.FC = () => { + setDownloadMenuEl(null)} + > + handleDownloadScript("sh")}> + For Mac / Linux (.sh) + + handleDownloadScript("bat")}> + For Windows (.bat) + + handleDownloadScript("txt")}> + URL list (.txt, advanced) + + @@ -1667,6 +1734,82 @@ const UpdatedDatasetDetailPage: React.FC = () => { key={`${previewIndex}-${previewOpen}`} // react will destroy the existing component and create a new one for mount /> + + {/* Post-download instructions. No auto-hide so users can read at + * their own pace; dismiss with the ✕ when finished. */} + setDownloadHint(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setDownloadHint(null)} + sx={{ maxWidth: 520 }} + > + {downloadHint === "sh" && ( + + + Downloaded the Mac / Linux script + + + To fetch your data files: + + +
  • Open Terminal
    • +
  • Go to the folder where the script was saved
    • +
  • + Run:{" "} + + bash <script-name>.sh + +
    • +     +     + )} + {downloadHint === "bat" && ( + + + Downloaded the Windows script + + + Open the folder where the script was saved and{" "} + double-click the .bat file. A command + window opens and the files download next to it. + + + )} + {downloadHint === "txt" && ( + + + Downloaded the URL list + + + In Terminal (Mac/Linux) or PowerShell (Windows), run:{" "} + + wget -i <file-name>.txt + + + + )} +     +     ); }; From 909cd36172343cf1b86492bae3c5d70174ce2558 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 20 May 2026 16:39:16 -0400 Subject: [PATCH 39/61] feat(sync): pull database list from registry --- backend/sync/incrementalSync.js | 48 +++++++-------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index b9e497a..55c5e90 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -8,34 +8,13 @@ const COUCHDB_URL = process.env.COUCHDB_URL || "https://neurojson.io:7777"; const CONCURRENCY = 5; // fetch database list dynamically from registry +// registry doc shape: { database: [{ id, name, ... }, ...] } async function getDatabases() { - try { - const response = await axios.get(`${COUCHDB_URL}/sys/registry`); - const databases = response.data - .map((db) => db.id) - .filter((id) => id && id !== "sys"); - console.log(`Found ${databases.length} databases in registry`); - return databases; - } catch (err) { - console.error("Failed to fetch registry:", err.message); - return [ - "openneuro", - "abide", - "abide2", - "datalad-registry", - "adhd200", - "bfnirs", - "mcx", - "mmc", - "ucl-4d-neonatal-head-model", - "unc-012-infant-atlas", - "unc-infant-cortical-surface-atlas", - "cotilab", - "emnist", - "nemo-bids", - "openfnirs", - ]; - } + const response = await axios.get(`${COUCHDB_URL}/sys/registry`); + const entries = response.data?.database || []; + const databases = entries.map((db) => db.id).filter(Boolean); + console.log(`Found ${databases.length} databases in registry`); + return databases; } // === Local ports of CouchDB _design/qq map functions === @@ -43,8 +22,7 @@ async function getDatabases() { // these drift silently. function transformDbinfo(doc) { - const txt = - doc["README"] || doc["README.md"] || doc["README.rst"] || ""; + const txt = doc["README"] || doc["README.md"] || doc["README.rst"] || ""; const rawtext = JSON.stringify(doc); const datainfo = doc["dataset_description.json"] || { Name: doc._id }; const subjlist = []; @@ -541,9 +519,7 @@ async function incrementalSync(dbname, lastSeq) { await processDatasetUpdate(dbname, change.id); } } catch (err) { - console.error( - ` ${dbname}/${change.id}: failed - ${err.message}` - ); + console.error(` ${dbname}/${change.id}: failed - ${err.message}`); } }) ); @@ -587,13 +563,7 @@ async function runSync() { console.log(new Date().toISOString()); console.log(`CouchDB: ${COUCHDB_URL}`); - // change to await getDatabases() when ready for full sync - const databases = [ - "bfnirs", // NIRS — .snirf, .jdb - "brainmeshlibrary", // mesh + atlas — .jmsh, .jnii (318 datasets) - "cotilab", // JData — small (6 datasets) - "abide", // BIDS MRI — .nii.gz, .tsv, .json (25 datasets) - ]; + const databases = await getDatabases(); console.log(`Databases: ${databases.length}`); for (const db of databases) { From 0cee8b6c221002d6d49c3f7748f0159d62906585 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 21 May 2026 11:43:56 -0400 Subject: [PATCH 40/61] feat(search): add dataset-level modality filter with AND/OR mode --- backend/src/controllers/couchdb.controller.js | 24 +++++ backend/sync/incrementalSync.js | 14 ++- src/pages/SearchPage.tsx | 97 ++++++++++++++++++- .../SearchPageFunctions/generateUiSchema.ts | 8 ++ .../SearchPageFunctions/searchformSchema.ts | 15 +++ 5 files changed, 151 insertions(+), 7 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index c990fe7..3700fc8 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -207,6 +207,30 @@ const searchAllDatabases = async (req, res) => { repl.modality = mod; } + // Dataset-level modality filter (multi-select + AND/OR). + // Queries json->'modality' on dbinfo rows, not subjects rows. + if (Array.isArray(f.modalities) && f.modalities.length > 0) { + const op = f.modality_mode === "and" ? " AND " : " OR "; + const parts = f.modalities.map((m, i) => { + repl[`dmod${i}`] = String(m); + return isSubjectSearch + ? `dsi.json->'modality' ? :dmod${i}` + : `json->'modality' ? :dmod${i}`; + }); + const condition = `(${parts.join(op)})`; + if (isSubjectSearch) { + where.push(`EXISTS ( + SELECT 1 FROM ioviews dsi + WHERE dsi.dbname = ioviews.dbname + AND dsi.dsname = ioviews.dsname + AND dsi.view = 'dbinfo' + AND ${condition} + )`); + } else { + where.push(condition); + } + } + // db / ds / subj filters if (isFilter(f.database)) { where.push(`dbname = :dbname`); diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 55c5e90..50fbad5 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -303,7 +303,15 @@ async function saveLastSeq(dbname, seq) { ); } +// Postgres jsonb rejects the null-byte escape with "unsupported Unicode +// escape sequence", so strip it from the serialized JSON before insert. +// Seen in openneuro README/TSV fields containing stray null bytes. +function safeStringify(obj) { + return JSON.stringify(obj).replace(/\\u0000/g, ""); +} + async function upsertIoview(dbname, dsname, subj, view, json, transaction) { + const payload = safeStringify(json); await sequelize.query( `INSERT INTO ioviews (dbname, dsname, subj, view, json, search_vector, updated_at) VALUES (:dbname, :dsname, :subj, :view, :json, to_tsvector('english', :text), NOW()) @@ -317,8 +325,8 @@ async function upsertIoview(dbname, dsname, subj, view, json, transaction) { dsname, subj: String(subj), view, - json: JSON.stringify(json), - text: JSON.stringify(json), + json: payload, + text: payload, }, transaction, } @@ -335,7 +343,7 @@ async function insertIolink(dbname, dsname, subj, view, json, transaction) { dsname, subj: String(subj), view, - json: JSON.stringify(json), + json: safeStringify(json), }, transaction, } diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 397e63a..ddd5137 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -16,6 +16,11 @@ import { Slider, Stack, TextField, + ToggleButton, + ToggleButtonGroup, + FormGroup, + FormControlLabel, + Checkbox, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -57,6 +62,80 @@ type RegistryItem = { const AGE_MIN_BOUND = 0; const AGE_MAX_BOUND = 100; +const DATASET_MODALITIES = [ + "anat", "func", "dwi", "fmap", "perf", + "meg", "eeg", "ieeg", "beh", "pet", + "micr", "nirs", "motion", "ephys", "atlas", + "JMesh", "JNIFTI", "JSNIRF", "JData", +]; + +const DatasetModalityFilterField = (props: any) => { + const ctx = props?.registry?.formContext as + | { formData: Record; setFormData: React.Dispatch>> } + | undefined; + if (!ctx) return null; + const { formData, setFormData } = ctx; + const selected: string[] = Array.isArray(formData.modalities) ? formData.modalities : []; + const mode: string = formData.modality_mode || "or"; + + const toggle = (code: string) => { + setFormData((prev) => { + const cur: string[] = Array.isArray(prev.modalities) ? prev.modalities : []; + const next = cur.includes(code) ? cur.filter((m) => m !== code) : [...cur, code]; + const updated = { ...prev }; + if (next.length === 0) { + delete updated.modalities; + delete updated.modality_mode; + } else { + updated.modalities = next; + if (!updated.modality_mode) updated.modality_mode = "or"; + } + return updated; + }); + }; + + const handleModeChange = (_: any, val: string | null) => { + if (!val) return; + setFormData((prev) => ({ ...prev, modality_mode: val })); + }; + + return ( + 0 ? "#e8f4fd" : "transparent" }}> + + Dataset modalities + + + {DATASET_MODALITIES.map((code) => ( + toggle(code)} + sx={{ py: 0.25 }} + /> + } + label={{code}} + sx={{ mr: 1 }} + /> + ))} + + {selected.length > 1 && ( + + + OR + AND + + + {mode === "and" ? "must have all selected" : "must have any selected"} + + + )} + + ); +}; + const AgeRangeSliderField = (props: any) => { const ctx = props?.registry?.formContext as | { @@ -239,10 +318,12 @@ const SearchPage: React.FC = () => { ([key, value]) => key !== "skip" && key !== "limit" && + key !== "modality_mode" && value !== undefined && value !== null && value !== "" && - value !== "any" + value !== "any" && + !(Array.isArray(value) && value.length === 0) ); useEffect(() => { @@ -381,6 +462,7 @@ const SearchPage: React.FC = () => { ), ageRangeSlider: AgeRangeSliderField, countRangePair: CountRangePairField, + datasetModalityFilter: DatasetModalityFilterField, }; // determine the results are subject-level or dataset-level @@ -685,10 +767,16 @@ const SearchPage: React.FC = () => { mt: 1, }} > - {activeFilters.map(([key, value]) => ( + {activeFilters.map(([key, value]) => { + let label = `${String(key)}: ${String(value)}`; + if (key === "modalities" && Array.isArray(value)) { + const mode = appliedFilters.modality_mode || "or"; + label = `modalities (${mode}): ${value.join(", ")}`; + } + return ( { } }} /> - ))} + ); + })} )} diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 152a056..46dc215 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -51,6 +51,9 @@ export const generateUiSchema = ( "database", "keyword", "file_type", // dataset-level: filters by file extensions in iolinks + "dataset_modality_filter", // dataset-level: modality multi-select + AND/OR + "modalities", + "modality_mode", "subject_filters_toggle", "age_range_slider", // top of subject filters — range slider for age "modality", @@ -106,6 +109,11 @@ export const generateUiSchema = ( }, } : datasetHiddenStyle, + dataset_modality_filter: showDatasetFilters + ? { "ui:field": "datasetModalityFilter" } + : datasetHiddenStyle, + modalities: invisibleStyle, + modality_mode: invisibleStyle, limit: invisibleStyle, skip: invisibleStyle, diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index e122725..3674cdd 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -134,6 +134,21 @@ export const baseSchema: JSONSchema7 = { items: { type: "string" }, uniqueItems: true, }, + dataset_modality_filter: { + type: "null", + title: "", + }, + modalities: { + type: "array", + title: "Dataset modalities", + items: { type: "string" }, + uniqueItems: true, + }, + modality_mode: { + type: "string", + title: "Modality match mode", + default: "or", + }, session_name: { title: "Session keywords", type: "string", From 601d90be7e248ed6e1b977883a6f456f715ab8e3 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 22 May 2026 10:10:44 -0400 Subject: [PATCH 41/61] feat(search): add tooltip to Subject-Level Filters explaining result type --- src/pages/SearchPage.tsx | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ddd5137..765e308 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -427,7 +427,7 @@ const SearchPage: React.FC = () => { // Create the "Subject-level Filters" button as a custom field const customFields = { subjectFiltersToggle: () => ( - + + + + + + ), datasetFiltersToggle: () => ( From 5aab8089f330c52d3ce592af07ba8f2152f04815 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 22 May 2026 10:24:34 -0400 Subject: [PATCH 42/61] fix(sync): reject malformed file extensions from CouchDB links view --- backend/sync/incrementalSync.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 50fbad5..766f1a5 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -310,6 +310,19 @@ function safeStringify(obj) { return JSON.stringify(obj).replace(/\\u0000/g, ""); } +// A valid file type is a dot-prefixed extension with no slashes and +// a reasonable length. Some CouchDB links view rows (e.g. openneuro) +// emit paths like ".0/libraries/FID-A/..." where the version number +// gets parsed as a fake extension — reject those. +function isValidFileType(ext) { + return ( + typeof ext === "string" && + ext.startsWith(".") && + !ext.includes("/") && + ext.length <= 20 + ); +} + async function upsertIoview(dbname, dsname, subj, view, json, transaction) { const payload = safeStringify(json); await sequelize.query( @@ -401,15 +414,18 @@ async function firstSync(dbname) { console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); const linkRows = await fetchView(dbname, "links"); + let linkCount = 0; for (const row of linkRows) { const fileType = row.key?.[0]; + if (!isValidFileType(fileType)) continue; const subjId = String(row.key?.[1] || ""); await insertIolink(dbname, row.id, subjId, fileType, { key: row.key, value: row.value, }); + linkCount++; } - console.log(` ${dbname}: links synced (${linkRows.length} rows)`); + console.log(` ${dbname}: links synced (${linkCount}/${linkRows.length} rows)`); } // === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === From f834a4740a5d912203d3b8ebe791049de23f3abe Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 25 May 2026 07:34:16 -0400 Subject: [PATCH 43/61] style(search): match dataset modality filter UI to file types field --- src/pages/SearchPage.tsx | 54 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 765e308..df49761 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -18,9 +18,7 @@ import { TextField, ToggleButton, ToggleButtonGroup, - FormGroup, - FormControlLabel, - Checkbox, + Autocomplete, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -78,10 +76,8 @@ const DatasetModalityFilterField = (props: any) => { const selected: string[] = Array.isArray(formData.modalities) ? formData.modalities : []; const mode: string = formData.modality_mode || "or"; - const toggle = (code: string) => { + const handleChange = (_: any, next: string[]) => { setFormData((prev) => { - const cur: string[] = Array.isArray(prev.modalities) ? prev.modalities : []; - const next = cur.includes(code) ? cur.filter((m) => m !== code) : [...cur, code]; const updated = { ...prev }; if (next.length === 0) { delete updated.modalities; @@ -100,27 +96,33 @@ const DatasetModalityFilterField = (props: any) => { }; return ( - 0 ? "#e8f4fd" : "transparent" }}> - - Dataset modalities - - - {DATASET_MODALITIES.map((code) => ( - toggle(code)} - sx={{ py: 0.25 }} - /> - } - label={{code}} - sx={{ mr: 1 }} + + + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + - ))} - + )} + /> {selected.length > 1 && ( From faf548ff118a7126fe8fa496bf83c11788cbe20a Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 26 May 2026 14:53:39 -0400 Subject: [PATCH 44/61] fix(search): resolve relative-path iolinks URLs for openneuro file links --- backend/src/controllers/couchdb.controller.js | 6 +- src/components/SearchPage/DatasetCard.tsx | 14 +++- yarn.lock | 78 ++++++++++--------- 3 files changed, 60 insertions(+), 38 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 3700fc8..ab12d0f 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -513,7 +513,11 @@ const getDatasetFilesManifest = async (req, res) => { } ); - const files = rows.filter((r) => r.url); + const resolveUrl = (url) => { + if (!url || url.startsWith("http")) return url; + return `https://neurojson.org/io/stat.cgi?action=get&db=${dbName}&doc=${dsName}&${url}`; + }; + const files = rows.filter((r) => r.url).map((r) => ({ ...r, url: resolveUrl(r.url) })); const urls = files.map((r) => r.url); const baseName = `${dbName}_${dsName}_${exts.join("_")}`; const extLabel = exts.join(", "); diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 4243119..7ecefae 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -59,6 +59,18 @@ interface DatasetCardProps { } /** ---------- utility helpers ---------- **/ +// Some iolinks records (older openneuro links view) store a relative path like +// "file=sub-01/anat/sub-01_T1w.nii&size=1" instead of a full stat.cgi URL. +// Reconstruct the full URL so the browser can follow it. +const resolveFileUrl = ( + dbname: string, + dsname: string, + url?: string +): string => { + if (!url) return ""; + if (url.startsWith("http")) return url; + return `https://neurojson.org/io/stat.cgi?action=get&db=${dbname}&doc=${dsname}&${url}`; +}; const normalize = (s: string) => s ?.replace(/[\u2018\u2019\u2032]/g, "'") // curly → straight @@ -504,7 +516,7 @@ const DatasetCard: React.FC = ({ return (
  • Date: Tue, 26 May 2026 17:32:27 -0400 Subject: [PATCH 45/61] feat(bids-converter): add public BIDS Converter page and navbar link --- src/components/NavBar/NavItems.tsx | 1 + src/components/Routes.tsx | 2 + src/pages/BidsConverterPage.tsx | 198 +++++++++++++++++++++++++++++ src/types/routes.enum.ts | 1 + 4 files changed, 202 insertions(+) create mode 100644 src/pages/BidsConverterPage.tsx diff --git a/src/components/NavBar/NavItems.tsx b/src/components/NavBar/NavItems.tsx index d2882e6..c102bcf 100644 --- a/src/components/NavBar/NavItems.tsx +++ b/src/components/NavBar/NavItems.tsx @@ -319,6 +319,7 @@ const NavItems: React.FC = () => { { text: "Wiki", url: "https://neurojson.org/Wiki" }, { text: "Search", url: RoutesEnum.SEARCH }, { text: "Databases", url: RoutesEnum.DATABASES }, + { text: "BIDS Converter", url: RoutesEnum.BIDS_CONVERTER }, ].map(({ text, url }) => ( {url?.startsWith("https") ? ( diff --git a/src/components/Routes.tsx b/src/components/Routes.tsx index 5983701..10b15e9 100644 --- a/src/components/Routes.tsx +++ b/src/components/Routes.tsx @@ -1,3 +1,4 @@ +import BidsConverterPage from "pages/BidsConverterPage"; import ScrollToTop from "./ScrollToTop"; import CompleteProfile from "./User/CompleteProfile"; import CollectionDetailPage from "./User/Dashboard/CollectionDetailPage"; @@ -68,6 +69,7 @@ const Routes = () => ( element={} /> } /> + } /> diff --git a/src/pages/BidsConverterPage.tsx b/src/pages/BidsConverterPage.tsx new file mode 100644 index 0000000..76de114 --- /dev/null +++ b/src/pages/BidsConverterPage.tsx @@ -0,0 +1,198 @@ +import DropZone from "components/User/Dashboard/DatasetOrganizer/DropZone"; +import FileTree from "components/User/Dashboard/DatasetOrganizer/FileTree"; +import LLMPanel from "components/User/Dashboard/DatasetOrganizer/LLMPanel"; +import { ArrowBack, GetApp, Psychology } from "@mui/icons-material"; +import { Box, Button, Typography, Alert } from "@mui/material"; +import { Colors } from "design/theme"; +import React, { useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { FileItem } from "redux/projects/types/projects.interface"; + +const BidsConverterPage: React.FC = () => { + const navigate = useNavigate(); + + const [files, setFiles] = useState([]); + const [selectedIds, setSelectedIds] = useState>(new Set()); + const [expandedIds, setExpandedIds] = useState>(new Set()); + const [showLLMPanel, setShowLLMPanel] = useState(false); + const [error, setError] = useState(null); + const [baseDirectoryPath, setBaseDirectoryPath] = useState(""); + const [evidenceBundle, setEvidenceBundle] = useState(null); + const [trioGenerated, setTrioGenerated] = useState(false); + + const updateFiles = (updater: React.SetStateAction) => + setFiles(updater); + + const updateSelectedIds = (updater: React.SetStateAction>) => + setSelectedIds(updater); + + const updateExpandedIds = (updater: React.SetStateAction>) => + setExpandedIds(updater); + + const updateBaseDirectoryPath = (path: string) => setBaseDirectoryPath(path); + + const handleExportJSON = () => { + const buildTree = (parentId: string | null): any => { + const children = files.filter((f) => f.parentId === parentId); + const result: any = {}; + children.forEach((child) => { + if (child.type === "folder" || child.type === "zip") { + result[child.name] = { + _type: child.type, + _sourcePath: baseDirectoryPath + ? `${baseDirectoryPath}/${child.sourcePath || child.name}`.replace(/\/+/g, "/") + : child.sourcePath || "", + _children: buildTree(child.id), + }; + } else { + const fileData: any = { + _type: "file", + _fileType: child.fileType || "other", + }; + if (child.sourcePath || baseDirectoryPath) { + fileData._sourcePath = baseDirectoryPath + ? `${baseDirectoryPath}/${child.sourcePath || child.name}`.replace(/\/+/g, "/") + : child.sourcePath; + } + if (child.isUserMeta) fileData._isUserMeta = true; + if (child.content) fileData._content = child.content; + if (child.contentType) fileData._contentType = child.contentType; + if (child.note) fileData._note = child.note; + result[child.name] = fileData; + } + }); + return result; + }; + + const exportData = { + _exportDate: new Date().toISOString(), + _totalFiles: files.length, + files: buildTree(null), + }; + + const blob = new Blob([JSON.stringify(exportData, null, 2)], { + type: "application/json", + }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "bids_converter_export.json"; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( + + {/* Header */} + + + + + BIDS Converter + + Organize and rename your dataset files into BIDS format + + + + + + + + + + + {error && ( + setError(null)} sx={{ m: 2 }}> + {error} + + )} + + {/* Main Content */} + + + + {showLLMPanel && ( + setShowLLMPanel(false)} + /> + )} + + + + + + ); +}; + +export default BidsConverterPage; diff --git a/src/types/routes.enum.ts b/src/types/routes.enum.ts index b37a84d..a774051 100644 --- a/src/types/routes.enum.ts +++ b/src/types/routes.enum.ts @@ -4,5 +4,6 @@ enum RoutesEnum { SEARCH = "/search", // New route for the search page ABOUT = "/about", // New route for the about page DASHBOARD = "/dashboard", + BIDS_CONVERTER = "/bids-converter", } export default RoutesEnum; From 11b77bde4464355bfd13077968679dadcb56a542 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 26 May 2026 18:00:15 -0400 Subject: [PATCH 46/61] feat(bids-converter): add mode selection dialog and private/save toggle bar --- src/pages/BidsConverterPage.tsx | 234 +++++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 6 deletions(-) diff --git a/src/pages/BidsConverterPage.tsx b/src/pages/BidsConverterPage.tsx index 76de114..2bbb3f0 100644 --- a/src/pages/BidsConverterPage.tsx +++ b/src/pages/BidsConverterPage.tsx @@ -1,15 +1,43 @@ import DropZone from "components/User/Dashboard/DatasetOrganizer/DropZone"; import FileTree from "components/User/Dashboard/DatasetOrganizer/FileTree"; import LLMPanel from "components/User/Dashboard/DatasetOrganizer/LLMPanel"; -import { ArrowBack, GetApp, Psychology } from "@mui/icons-material"; -import { Box, Button, Typography, Alert } from "@mui/material"; +import UserLogin from "components/User/UserLogin"; +import UserSignup from "components/User/UserSignup"; +import { + ArrowBack, + GetApp, + Psychology, + LockOutlined, + CloudUpload, +} from "@mui/icons-material"; +import { + Box, + Button, + Typography, + Alert, + Dialog, + DialogContent, + DialogTitle, + ToggleButton, + ToggleButtonGroup, +} from "@mui/material"; import { Colors } from "design/theme"; -import React, { useState } from "react"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useState, useEffect } from "react"; import { useNavigate } from "react-router-dom"; +import { AuthSelector } from "redux/auth/auth.selector"; import { FileItem } from "redux/projects/types/projects.interface"; +type Mode = "private" | "save"; + const BidsConverterPage: React.FC = () => { const navigate = useNavigate(); + const { isLoggedIn } = useAppSelector(AuthSelector); + + const [modeChosen, setModeChosen] = useState(false); + const [mode, setMode] = useState("private"); + const [loginOpen, setLoginOpen] = useState(false); + const [signupOpen, setSignupOpen] = useState(false); const [files, setFiles] = useState([]); const [selectedIds, setSelectedIds] = useState>(new Set()); @@ -20,15 +48,42 @@ const BidsConverterPage: React.FC = () => { const [evidenceBundle, setEvidenceBundle] = useState(null); const [trioGenerated, setTrioGenerated] = useState(false); + // After login succeeds in save mode, redirect to dashboard to create a project + useEffect(() => { + if (isLoggedIn && mode === "save") { + navigate("/dashboard"); + } + }, [isLoggedIn, mode, navigate]); + + const handleChoosePrivate = () => { + setMode("private"); + setModeChosen(true); + }; + + const handleChooseSave = () => { + setMode("save"); + if (isLoggedIn) { + navigate("/dashboard"); + } else { + setLoginOpen(true); + } + }; + + const handleModeBarChange = (_: React.MouseEvent, next: Mode | null) => { + if (!next || next === mode) return; + if (next === "save") { + handleChooseSave(); + } else { + setMode("private"); + } + }; + const updateFiles = (updater: React.SetStateAction) => setFiles(updater); - const updateSelectedIds = (updater: React.SetStateAction>) => setSelectedIds(updater); - const updateExpandedIds = (updater: React.SetStateAction>) => setExpandedIds(updater); - const updateBaseDirectoryPath = (path: string) => setBaseDirectoryPath(path); const handleExportJSON = () => { @@ -148,6 +203,63 @@ const BidsConverterPage: React.FC = () => { + {/* Mode indicator bar */} + + + + + Private Mode + + + + Save to Account + + + + + {mode === "private" + ? "Files are processed locally. Nothing is uploaded. All data is lost when you close this page." + : "Log in to save your work to a project on your account."} + + + {error && ( setError(null)} sx={{ m: 2 }}> {error} @@ -191,6 +303,116 @@ const BidsConverterPage: React.FC = () => { setExpandedIds={updateExpandedIds} /> + + {/* Welcome dialog — shown on first load before user starts working */} + + + How would you like to use BIDS Converter? + + + + {/* Private Mode card */} + + + + Private Mode + + + Work entirely in your browser. No files are uploaded to any + server. All data will be lost when you close the page. + + + + {/* Save to Account card */} + + + + Save to Account + + + Log in to save your work to a project. You can resume it any + time from your dashboard. + + + + + + + { + setLoginOpen(false); + // If user closes login without logging in, fall back to private mode + if (!isLoggedIn) { + setMode("private"); + setModeChosen(true); + } + }} + onSwitchToSignup={() => { + setLoginOpen(false); + setSignupOpen(true); + }} + /> + { + setSignupOpen(false); + if (!isLoggedIn) { + setMode("private"); + setModeChosen(true); + } + }} + onSwitchToLogin={() => { + setSignupOpen(false); + setLoginOpen(true); + }} + /> ); }; From 7dbb33555d39d2d0f7819ce8b0e5c80d3a236f44 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 27 May 2026 11:47:49 -0400 Subject: [PATCH 47/61] feat(dashboard): open Projects tab directly from BIDS Converter redirect --- src/components/User/UserDashboard.tsx | 20 +++++++++++++++++++- src/pages/BidsConverterPage.tsx | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/components/User/UserDashboard.tsx b/src/components/User/UserDashboard.tsx index 21d4e53..42eab48 100644 --- a/src/components/User/UserDashboard.tsx +++ b/src/components/User/UserDashboard.tsx @@ -24,6 +24,7 @@ import { import { Colors } from "design/theme"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useState } from "react"; +import { useLocation, useNavigate } from "react-router-dom"; import { AuthSelector } from "redux/auth/auth.selector"; interface TabPanelProps { @@ -48,12 +49,29 @@ function TabPanel(props: TabPanelProps) { ); } +const TAB_INDEX: Record = { + profile: 0, + security: 1, + collections: 2, + liked: 3, + projects: 4, + settings: 5, +}; + +const TAB_NAME = Object.fromEntries( + Object.entries(TAB_INDEX).map(([k, v]) => [v, k]) +) as Record; + const UserDashboard: React.FC = () => { - const [tabValue, setTabValue] = useState(0); + const location = useLocation(); + const navigate = useNavigate(); + const tabParam = new URLSearchParams(location.search).get("tab") ?? ""; + const [tabValue, setTabValue] = useState(TAB_INDEX[tabParam] ?? 0); const { user } = useAppSelector(AuthSelector); const handleTabChange = (event: React.SyntheticEvent, newValue: number) => { setTabValue(newValue); + navigate(`/dashboard?tab=${TAB_NAME[newValue]}`, { replace: true }); }; if (!user) { diff --git a/src/pages/BidsConverterPage.tsx b/src/pages/BidsConverterPage.tsx index 2bbb3f0..1c57d16 100644 --- a/src/pages/BidsConverterPage.tsx +++ b/src/pages/BidsConverterPage.tsx @@ -51,7 +51,7 @@ const BidsConverterPage: React.FC = () => { // After login succeeds in save mode, redirect to dashboard to create a project useEffect(() => { if (isLoggedIn && mode === "save") { - navigate("/dashboard"); + navigate("/dashboard?tab=projects"); } }, [isLoggedIn, mode, navigate]); @@ -63,7 +63,7 @@ const BidsConverterPage: React.FC = () => { const handleChooseSave = () => { setMode("save"); if (isLoggedIn) { - navigate("/dashboard"); + navigate("/dashboard?tab=projects"); } else { setLoginOpen(true); } From 71e5ffb0eba4571ba65ee1bf6244679a874a80ec Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 27 May 2026 12:00:44 -0400 Subject: [PATCH 48/61] feat(bids-converter): add private mode visual indicator with slate grey background --- src/pages/BidsConverterPage.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pages/BidsConverterPage.tsx b/src/pages/BidsConverterPage.tsx index 1c57d16..1f27008 100644 --- a/src/pages/BidsConverterPage.tsx +++ b/src/pages/BidsConverterPage.tsx @@ -142,7 +142,9 @@ const BidsConverterPage: React.FC = () => { display: "flex", flexDirection: "column", height: "100vh", - background: "linear-gradient(180deg,#f6f7fb 0%, #aeb6e8 100%)", + background: mode === "private" + ? "linear-gradient(180deg, #eceff1 0%, #90a4ae 100%)" + : "linear-gradient(180deg,#f6f7fb 0%, #aeb6e8 100%)", }} > {/* Header */} @@ -213,7 +215,7 @@ const BidsConverterPage: React.FC = () => { display: "flex", alignItems: "center", gap: 2, - backgroundColor: "white", + backgroundColor: mode === "private" ? "#eceff1" : "white", }} > Date: Wed, 27 May 2026 15:11:48 -0400 Subject: [PATCH 49/61] feat(bids-converter): improve UI labels, tooltips, and private mode UX --- .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 83 ++++++++++++++----- .../User/Dashboard/DatasetOrganizer/index.tsx | 3 +- src/pages/BidsConverterPage.tsx | 50 ++++++++++- 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index 9658ec6..1e49428 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -13,6 +13,7 @@ import { Download, AutoAwesome, DriveFileMove, + InfoOutlined, } from "@mui/icons-material"; import { Box, @@ -27,6 +28,7 @@ import { CircularProgress, IconButton, Alert, + Tooltip, } from "@mui/material"; import { Colors } from "design/theme"; import { dump as yamlDump } from "js-yaml"; @@ -46,6 +48,7 @@ interface LLMPanelProps { setTrioGenerated: (value: boolean) => void; // ✅ Add updateFiles: (updater: React.SetStateAction) => void; // ✅ Add onClose: () => void; + isPrivateMode?: boolean; } interface LLMProvider { @@ -77,28 +80,27 @@ const llmProviders: Record = { baseUrl: "https://api.groq.com/openai/v1/chat/completions", models: [ { id: "llama-3.3-70b-versatile", name: "Llama 3.3 70B" }, + { id: "llama-3.1-70b-versatile", name: "Llama 3.1 70B" }, { id: "llama-3.1-8b-instant", name: "Llama 3.1 8B (Fast)" }, - { id: "mixtral-8x7b-32768", name: "Mixtral 8x7B" }, + { id: "gemma2-9b-it", name: "Gemma 2 9B" }, ], }, openrouter: { name: "OpenRouter (Free models available)", baseUrl: "https://openrouter.ai/api/v1/chat/completions", models: [ - { - id: "meta-llama/llama-3.1-8b-instruct:free", - name: "Llama 3.1 8B (Free)", - }, - { id: "google/gemma-2-9b-it:free", name: "Gemma 2 9B (Free)" }, - { id: "mistralai/mistral-7b-instruct:free", name: "Mistral 7B (Free)" }, + { id: "meta-llama/llama-3.3-70b-instruct:free", name: "Llama 3.3 70B (Free)" }, + { id: "google/gemma-3n-e4b-it:free", name: "Gemma 3n 4B (Free)" }, + { id: "mistralai/mistral-small-3.2-24b-instruct:free", name: "Mistral Small 3.2 24B (Free)" }, ], }, anthropic: { name: "Anthropic Claude (Paid)", baseUrl: "https://api.anthropic.com/v1/messages", models: [ - { id: "claude-sonnet-4-20250514", name: "Claude Sonnet 4" }, - { id: "claude-3-5-haiku-20241022", name: "Claude 3.5 Haiku" }, + { id: "claude-opus-4-7", name: "Claude Opus 4.7" }, + { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, + { id: "claude-haiku-4-5-20251001", name: "Claude Haiku 4.5" }, ], isAnthropic: true, }, @@ -106,8 +108,11 @@ const llmProviders: Record = { name: "OpenAI (Paid)", baseUrl: "https://api.openai.com/v1/chat/completions", models: [ - { id: "gpt-4o-mini", name: "GPT-4o Mini" }, + { id: "gpt-5.5", name: "GPT-5.5" }, + { id: "gpt-5.4", name: "GPT-5.4" }, + { id: "gpt-5.4-mini", name: "GPT-5.4 Mini" }, { id: "gpt-4o", name: "GPT-4o" }, + { id: "gpt-4o-mini", name: "GPT-4o Mini" }, ], }, }; @@ -122,9 +127,10 @@ const LLMPanel: React.FC = ({ setTrioGenerated, // ✅ Add updateFiles, // ✅ Add onClose, + isPrivateMode = false, }) => { - const [provider, setProvider] = useState("ollama"); - const [model, setModel] = useState("qwen3-coder-next:latest"); + const [provider, setProvider] = useState(isPrivateMode ? "groq" : "ollama"); + const [model, setModel] = useState(isPrivateMode ? "llama-3.3-70b-versatile" : "qwen3-coder-next:latest"); // const [ollamaUrl, setOllamaUrl] = useState( // "http://jin.neu.edu:11434" // ); @@ -1208,7 +1214,7 @@ const LLMPanel: React.FC = ({ }); updateFiles((prev) => [...prev, ...outputFiles]); - setStatus("✓ Saved to VFS. Click 'Save Changes' to persist to database."); + setStatus(isPrivateMode ? "✓ Saved to VFS." : "✓ Saved to VFS. Click 'Save Changes' to persist to database."); }; // const handleSaveZip = async () => { // const zip = new JSZip(); @@ -1350,7 +1356,38 @@ const LLMPanel: React.FC = ({ sx={{ display: "flex", alignItems: "center", gap: 1 }} > - AI-Generated BIDS Conversion Script + AI Assistant + + + + + @@ -1379,11 +1416,13 @@ const LLMPanel: React.FC = ({ setModel(llmProviders[e.target.value].models[0].id); }} > - {Object.entries(llmProviders).map(([key, p]) => ( - - {p.name} - - ))} + {Object.entries(llmProviders) + .filter(([key]) => !(isPrivateMode && key === "ollama")) + .map(([key, p]) => ( + + {p.name} + + ))} @@ -1567,8 +1606,8 @@ const LLMPanel: React.FC = ({ {generatingTrio ? "Generating..." : trioGenerated - ? "✓ 2. Generate BIDS Trio" - : "2. Generate BIDS Trio"} + ? "✓ 2. Generate BIDS Metadata Files" + : "2. Generate BIDS Metadata Files"} {/* = ({ "&.Mui-disabled": { background: "#e0e0e0", color: "#9e9e9e" }, }} > - {loading ? "Generating..." : "3. Generate BIDSPlan.yaml"} + {loading ? "Generating..." : "3. Generate Conversion Package"} {/* {/* - BIDS Converter + + BIDS Converter + + How to use: + +
  • Drop your dataset files into the workspace.
    • +
  • Enter the number of subjects, modality, and base directory path.
    • +
  • The AI will analyze your files and generate a BIDS conversion plan.
    • +
  • Download and run the script locally to reorganize your data into BIDS format.
    • + +     + } + placement="bottom-start" + arrow + componentsProps={{ + tooltip: { + sx: { + backgroundColor: "white", + color: Colors.darkPurple, + border: `1px solid ${Colors.lightGray}`, + boxShadow: 3, + fontSize: "0.875rem", + lineHeight: 1.5, + p: 1.5, + maxWidth: 320, + }, + }, + arrow: { + sx: { + color: "white", + "&::before": { border: `1px solid ${Colors.lightGray}` }, + }, + }, + }} + > + + + + +     Organize and rename your dataset files into BIDS format @@ -180,14 +224,13 @@ const BidsConverterPage: React.FC = () => { variant="contained" startIcon={} onClick={() => setShowLLMPanel(!showLLMPanel)} - disabled={files.length === 0} sx={{ backgroundColor: Colors.purple, color: Colors.lightGray, "&:hover": { backgroundColor: Colors.purple, border: "none" }, }} > - Generate BIDS Plan + AI Assistant - {/* */}     @@ -1755,11 +1745,13 @@ const LLMPanel: React.FC = ({ color: "#d4d4d4", }} > - {/* {generatedScript || - 'Configure your LLM provider and click "Generate Script"...'} */} - {bidsPlan || - generatedScript || - 'Configure your LLM provider and click "Generate BIDSPlan.yaml"...'} + {bidsPlan || generatedScript || ( + + {status && !error + ? status + : 'Fill in the fields on the left and follow the steps to generate your conversion package...'} + + )} From cf2e445020ad22e35524d506a43976fb4c666694 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 27 May 2026 16:11:17 -0400 Subject: [PATCH 51/61] fix(llm-panel): clear bidsPlan on step 2 re-run so progress shows in output panel --- src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx | 1 + src/components/User/Dashboard/DatasetOrganizer/index.tsx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index bb8d69a..6b3c007 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -218,6 +218,7 @@ const LLMPanel: React.FC = ({ setAbortController(controller); setGeneratingTrio(true); setError(null); + setBidsPlan(""); setStatus("Generating BIDS trio files..."); try { diff --git a/src/components/User/Dashboard/DatasetOrganizer/index.tsx b/src/components/User/Dashboard/DatasetOrganizer/index.tsx index 39c2f45..5a361a9 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/index.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/index.tsx @@ -282,7 +282,7 @@ const DatasetOrganizer: React.FC = () => { > AI Assistant - {/* */} + + */} @@ -1728,9 +1733,13 @@ const LLMPanel: React.FC = ({ startIcon={} onClick={handleSaveZip} disabled={!bidsPlan || !trioGenerated} - sx={{ color: Colors.darkGreen, borderColor: Colors.darkGreen }} + sx={{ + color: Colors.purple, + borderColor: Colors.purple, + "&:hover": { borderColor: Colors.purple }, + }} > - Preview in File Tree + Preview Conversion Package in File Tree From 60f51ade5d684d0c1eafcd5c647ac7a5f1d7aded Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 27 May 2026 17:56:39 -0400 Subject: [PATCH 53/61] feat(ollama): require authentication on Ollama proxy endpoints --- backend/src/routes/ollama.routes.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/routes/ollama.routes.js b/backend/src/routes/ollama.routes.js index ff1fd94..38bdcb4 100644 --- a/backend/src/routes/ollama.routes.js +++ b/backend/src/routes/ollama.routes.js @@ -1,8 +1,9 @@ const express = require("express"); const router = express.Router(); const { proxyChat, getTags } = require("../controllers/ollama.controller"); +const { requireAuth } = require("../middleware/auth.middleware"); -router.post("/chat", proxyChat); -router.get("/tags", getTags); +router.post("/chat", requireAuth, proxyChat); +router.get("/tags", requireAuth, getTags); module.exports = router; From 68218d5b000149565f7f86f05aaf775235376ef9 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 27 May 2026 18:11:41 -0400 Subject: [PATCH 54/61] feat(ollama): add public endpoint with 10 requests/day IP rate limit --- backend/package-lock.json | 26 ++++++++++++++++++---- backend/package.json | 1 + backend/src/routes/ollama.public.routes.js | 17 ++++++++++++++ backend/src/server.js | 2 ++ 4 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 backend/src/routes/ollama.public.routes.js diff --git a/backend/package-lock.json b/backend/package-lock.json index 70f2f74..9bb2c58 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,6 +15,7 @@ "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^5.1.0", + "express-rate-limit": "^8.5.2", "jsonwebtoken": "^9.0.2", "nanoid": "^3.3.11", "nodemailer": "^7.0.11", @@ -1181,6 +1182,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -1742,11 +1761,10 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", - "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", "license": "MIT", - "optional": true, "engines": { "node": ">= 12" } diff --git a/backend/package.json b/backend/package.json index fa156b3..45f386a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -29,6 +29,7 @@ "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^5.1.0", + "express-rate-limit": "^8.5.2", "jsonwebtoken": "^9.0.2", "nanoid": "^3.3.11", "nodemailer": "^7.0.11", diff --git a/backend/src/routes/ollama.public.routes.js b/backend/src/routes/ollama.public.routes.js new file mode 100644 index 0000000..e9fc0f1 --- /dev/null +++ b/backend/src/routes/ollama.public.routes.js @@ -0,0 +1,17 @@ +const express = require("express"); +const router = express.Router(); +const rateLimit = require("express-rate-limit"); +const { proxyChat, getTags } = require("../controllers/ollama.controller"); + +const dailyLimit = rateLimit({ + windowMs: 24 * 60 * 60 * 1000, // 24 hours + max: 10, + standardHeaders: true, + legacyHeaders: false, + message: { error: "Daily request limit reached. You can send up to 10 requests per day from this IP." }, +}); + +router.post("/chat", dailyLimit, proxyChat); +router.get("/tags", dailyLimit, getTags); + +module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index 5e6cb6c..d0d043d 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -14,6 +14,7 @@ const datasetsRoutes = require("./routes/datasets.routes"); const collectionRoutes = require("./routes/collection.route"); const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); +const ollamaPublicRoutes = require("./routes/ollama.public.routes"); const app = express(); const PORT = process.env.PORT || 5000; @@ -51,6 +52,7 @@ app.use("/api/v1/datasets", datasetsRoutes); app.use("/api/v1/collections", collectionRoutes); app.use("/api/v1/projects", projectRoutes); app.use("/api/v1/ollama", ollamaRoutes); +app.use("/api/v1/ollama-public", ollamaPublicRoutes); // health check endpoint app.get("/api/health", async (req, res) => { From 1aa509af696ef0b92f28489b31b5f08276a7f014 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 29 May 2026 11:31:07 -0400 Subject: [PATCH 55/61] feat(llm-panel): add local Ollama provider for private mode with configurable URL --- .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index a37ba9e..d858c35 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -73,7 +73,18 @@ const llmProviders: Record = { { id: "qwen2.5-coder:7b", name: "Qwen 2.5 Coder 7B" }, ], noApiKey: true, - // customUrl: true, + }, + "local-ollama": { + name: "Ollama (Your Local Machine)", + baseUrl: "http://localhost:11434/v1/chat/completions", + models: [ + { id: "llama3.2:latest", name: "Llama 3.2" }, + { id: "llama3.1:latest", name: "Llama 3.1" }, + { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder" }, + { id: "mistral:latest", name: "Mistral" }, + { id: "gemma3:latest", name: "Gemma 3" }, + ], + noApiKey: true, }, groq: { name: "Groq (Free API Key - 14,400 req/day)", @@ -129,8 +140,9 @@ const LLMPanel: React.FC = ({ onClose, isPrivateMode = false, }) => { - const [provider, setProvider] = useState(isPrivateMode ? "groq" : "ollama"); - const [model, setModel] = useState(isPrivateMode ? "llama-3.3-70b-versatile" : "qwen3-coder-next:latest"); + const [provider, setProvider] = useState(isPrivateMode ? "local-ollama" : "ollama"); + const [model, setModel] = useState(isPrivateMode ? "llama3.2:latest" : "qwen3-coder-next:latest"); + const [localOllamaUrl, setLocalOllamaUrl] = useState("http://localhost:11434"); // const [ollamaUrl, setOllamaUrl] = useState( // "http://jin.neu.edu:11434" // ); @@ -160,7 +172,9 @@ const LLMPanel: React.FC = ({ provider, model, apiKey, - baseUrl: currentProvider.baseUrl, + baseUrl: provider === "local-ollama" + ? `${localOllamaUrl}/v1/chat/completions` + : currentProvider.baseUrl, isAnthropic: currentProvider.isAnthropic, noApiKey: currentProvider.noApiKey, }); @@ -1421,7 +1435,7 @@ const LLMPanel: React.FC = ({ }} > {Object.entries(llmProviders) - .filter(([key]) => !(isPrivateMode && key === "ollama")) + .filter(([key]) => isPrivateMode ? key !== "ollama" : key !== "local-ollama") .map(([key, p]) => ( {p.name} @@ -1445,17 +1459,17 @@ const LLMPanel: React.FC = ({ - {/* Ollama Server URL field */} - {/* {provider === "ollama" && ( + {provider === "local-ollama" && ( setOllamaUrl(e.target.value)} + label="Ollama URL" + value={localOllamaUrl} + onChange={(e) => setLocalOllamaUrl(e.target.value)} placeholder="http://localhost:11434" + helperText="Default port is 11434. Change if your Ollama runs on a different port." sx={{ mb: 2 }} /> - )} */} + )} {/* Base Directory Path field (shows for ALL providers) */} Date: Fri, 29 May 2026 11:42:20 -0400 Subject: [PATCH 56/61] feat(navbar): add AutoBIDSify dropdown with GitHub link and web app; update dialog title and description --- src/components/NavBar/NavItems.tsx | 92 ++++++++++++++++++++++++++++-- src/pages/BidsConverterPage.tsx | 5 +- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/components/NavBar/NavItems.tsx b/src/components/NavBar/NavItems.tsx index c102bcf..bfbb9b3 100644 --- a/src/components/NavBar/NavItems.tsx +++ b/src/components/NavBar/NavItems.tsx @@ -35,11 +35,13 @@ const NavItems: React.FC = () => { const [signupOpen, setSignupOpen] = useState(false); // Resources dropdown state - const [resourcesAnchor, setResourcesAnchor] = useState( - null - ); + const [resourcesAnchor, setResourcesAnchor] = useState(null); const resourcesOpen = Boolean(resourcesAnchor); + // AutoBIDSify dropdown state + const [autobidsifyAnchor, setAutobidsifyAnchor] = useState(null); + const autobidsifyOpen = Boolean(autobidsifyAnchor); + const handleLogout = () => { dispatch(logoutUser()); navigate("/"); @@ -53,6 +55,14 @@ const NavItems: React.FC = () => { setResourcesAnchor(null); }; + const handleAutobidsifyClick = (event: React.MouseEvent) => { + setAutobidsifyAnchor(event.currentTarget); + }; + + const handleAutobidsifyClose = () => { + setAutobidsifyAnchor(null); + }; + const resourcesMenu = [ { category: "Converter", @@ -319,7 +329,6 @@ const NavItems: React.FC = () => { { text: "Wiki", url: "https://neurojson.org/Wiki" }, { text: "Search", url: RoutesEnum.SEARCH }, { text: "Databases", url: RoutesEnum.DATABASES }, - { text: "BIDS Converter", url: RoutesEnum.BIDS_CONVERTER }, ].map(({ text, url }) => ( {url?.startsWith("https") ? ( @@ -379,6 +388,38 @@ const NavItems: React.FC = () => { ))} + {/* AutoBIDSify Dropdown */} + + + + AutoBIDSify + + + + + {/* Resources Dropdown */} { ))} + + {/* AutoBIDSify Dropdown Menu */} + + { + window.open("https://github.com/COTILab/autobidsify", "_blank"); + handleAutobidsifyClose(); + }} + sx={{ + fontSize: "0.9rem", + color: Colors.white, + "&:hover": { bgcolor: Colors.purpleGrey, color: Colors.darkPurple }, + }} + > + AutoBIDSify (GitHub) + + { + navigate(RoutesEnum.BIDS_CONVERTER); + handleAutobidsifyClose(); + }} + sx={{ + fontSize: "0.9rem", + color: Colors.white, + "&:hover": { bgcolor: Colors.purpleGrey, color: Colors.darkPurple }, + }} + > + AutoBIDSify Web + + + setLoginOpen(false)} diff --git a/src/pages/BidsConverterPage.tsx b/src/pages/BidsConverterPage.tsx index 70b4a46..b528567 100644 --- a/src/pages/BidsConverterPage.tsx +++ b/src/pages/BidsConverterPage.tsx @@ -353,9 +353,12 @@ const BidsConverterPage: React.FC = () => { {/* Welcome dialog — shown on first load before user starts working */} - How would you like to use BIDS Converter? + How would you like to use AutoBIDSify? + + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format. + Date: Fri, 29 May 2026 11:59:49 -0400 Subject: [PATCH 57/61] feat(autobidsify): rebrand BIDS Converter to AutoBIDSify with navbar dropdown, updated labels, and GitHub links --- .../User/Dashboard/DatasetOrganizer/index.tsx | 51 +++++++++++++++++++ src/components/User/Dashboard/ProjectsTab.tsx | 4 +- src/pages/BidsConverterPage.tsx | 22 ++++++-- src/types/routes.enum.ts | 2 +- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/index.tsx b/src/components/User/Dashboard/DatasetOrganizer/index.tsx index 5a361a9..796fbaa 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/index.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/index.tsx @@ -14,6 +14,8 @@ import { DialogContent, DialogActions, DialogContentText, + Chip, + Tooltip, } from "@mui/material"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; @@ -263,6 +265,55 @@ const DatasetOrganizer: React.FC = () => { {currentProject.description} )} + + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format. + window.open("https://github.com/COTILab/autobidsify", "_blank")} + > + Learn more + + + } + placement="bottom-start" + arrow + componentsProps={{ + tooltip: { + sx: { + backgroundColor: "white", + color: Colors.darkPurple, + border: `1px solid ${Colors.lightGray}`, + boxShadow: 3, + fontSize: "0.875rem", + p: 1.5, + maxWidth: 320, + }, + }, + arrow: { + sx: { + color: "white", + "&::before": { border: `1px solid ${Colors.lightGray}` }, + }, + }, + }} + > + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ + mt: 0.5, + backgroundColor: Colors.purple, + color: Colors.white, + fontSize: "0.7rem", + cursor: "pointer", + "&:hover": { backgroundColor: Colors.secondaryPurple }, + }} + /> + diff --git a/src/components/User/Dashboard/ProjectsTab.tsx b/src/components/User/Dashboard/ProjectsTab.tsx index 6095c9b..ceaa91b 100644 --- a/src/components/User/Dashboard/ProjectsTab.tsx +++ b/src/components/User/Dashboard/ProjectsTab.tsx @@ -199,10 +199,10 @@ const ProjectsTab: React.FC = ({ userId }) => { > - Dataset Organizer Projects + AutoBIDSify Projects - Organize and convert your neuroimaging datasets to BIDS format + Organize and convert your neuroimaging datasets to BIDS format using AutoBIDSify - BIDS Converter + AutoBIDSify @@ -214,7 +214,15 @@ const BidsConverterPage: React.FC = () => { - Organize and rename your dataset files into BIDS format + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format.{" "} + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ color: Colors.purple, cursor: "pointer", textDecoration: "underline" }} + > + Learn more + @@ -357,7 +365,15 @@ const BidsConverterPage: React.FC = () => { - An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format. + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format.{" "} + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ color: Colors.purple, cursor: "pointer", textDecoration: "underline" }} + > + Learn more + Date: Fri, 29 May 2026 12:14:11 -0400 Subject: [PATCH 58/61] feat(llm-panel): add local AI support for Ollama, LM Studio, and Jan with custom model input --- .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index d858c35..84e1a7a 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -75,14 +75,19 @@ const llmProviders: Record = { noApiKey: true, }, "local-ollama": { - name: "Ollama (Your Local Machine)", + name: "Local AI (Ollama / LM Studio / Jan)", baseUrl: "http://localhost:11434/v1/chat/completions", models: [ - { id: "llama3.2:latest", name: "Llama 3.2" }, - { id: "llama3.1:latest", name: "Llama 3.1" }, - { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder" }, - { id: "mistral:latest", name: "Mistral" }, - { id: "gemma3:latest", name: "Gemma 3" }, + { id: "llama3.2:latest", name: "Llama 3.2 (Ollama)" }, + { id: "llama3.1:latest", name: "Llama 3.1 (Ollama)" }, + { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (Ollama)" }, + { id: "mistral:latest", name: "Mistral (Ollama)" }, + { id: "gemma3:latest", name: "Gemma 3 (Ollama)" }, + { id: "llama-3.2-3b-instruct", name: "Llama 3.2 3B (LM Studio)" }, + { id: "llama-3.1-8b-instruct", name: "Llama 3.1 8B (LM Studio)" }, + { id: "mistral-7b-instruct-v0.3", name: "Mistral 7B (LM Studio)" }, + { id: "llama3.2:3b", name: "Llama 3.2 3B (Jan)" }, + { id: "mistral:7b", name: "Mistral 7B (Jan)" }, ], noApiKey: true, }, @@ -1459,6 +1464,25 @@ const LLMPanel: React.FC = ({ + {provider === "local-ollama" && ( + { + if (e.target.value.trim()) setModel(e.target.value.trim()); + }} + /> + )} + + {isPrivateMode && provider !== "local-ollama" && ( + + Your file information will be sent to {currentProvider.name}, an external AI service. Switch to Local AI (Ollama / LM Studio / Jan) to keep everything local. + + )} + {provider === "local-ollama" && ( = ({ value={localOllamaUrl} onChange={(e) => setLocalOllamaUrl(e.target.value)} placeholder="http://localhost:11434" - helperText="Default port is 11434. Change if your Ollama runs on a different port." + helperText="Ollama: port 11434 · LM Studio: port 1234 · Jan: port 1337" sx={{ mb: 2 }} /> )} From 1dfc255dbdef999b7db756ad0d737149a21f0e22 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 29 May 2026 12:46:13 -0400 Subject: [PATCH 59/61] feat(ollama): use qwen3.6:27b on server for LLM, hide model selector --- backend/src/controllers/ollama.controller.js | 5 +++-- backend/src/routes/ollama.public.routes.js | 2 +- backend/src/routes/ollama.routes.js | 2 +- .../Dashboard/DatasetOrganizer/LLMPanel.tsx | 22 +++++++++++++------ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/backend/src/controllers/ollama.controller.js b/backend/src/controllers/ollama.controller.js index 84bb3bb..fe0c4e8 100644 --- a/backend/src/controllers/ollama.controller.js +++ b/backend/src/controllers/ollama.controller.js @@ -1,12 +1,13 @@ const OLLAMA_BASE_URL = "http://jin.neu.edu:11434"; +const OLLAMA_MODEL = "qwen3.6:27b"; const proxyChat = async (req, res) => { - console.log("🟣 [Ollama] proxyChat hit — model:", req.body.model); + console.log("🟣 [Ollama] proxyChat hit — model:", OLLAMA_MODEL); try { const response = await fetch(`${OLLAMA_BASE_URL}/v1/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(req.body), + body: JSON.stringify({ ...req.body, model: OLLAMA_MODEL }), }); const data = await response.json(); diff --git a/backend/src/routes/ollama.public.routes.js b/backend/src/routes/ollama.public.routes.js index e9fc0f1..b8981a0 100644 --- a/backend/src/routes/ollama.public.routes.js +++ b/backend/src/routes/ollama.public.routes.js @@ -12,6 +12,6 @@ const dailyLimit = rateLimit({ }); router.post("/chat", dailyLimit, proxyChat); -router.get("/tags", dailyLimit, getTags); +// router.get("/tags", dailyLimit, getTags); module.exports = router; diff --git a/backend/src/routes/ollama.routes.js b/backend/src/routes/ollama.routes.js index 38bdcb4..be86d75 100644 --- a/backend/src/routes/ollama.routes.js +++ b/backend/src/routes/ollama.routes.js @@ -4,6 +4,6 @@ const { proxyChat, getTags } = require("../controllers/ollama.controller"); const { requireAuth } = require("../middleware/auth.middleware"); router.post("/chat", requireAuth, proxyChat); -router.get("/tags", requireAuth, getTags); +// router.get("/tags", requireAuth, getTags); module.exports = router; diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index 84e1a7a..c574bf5 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -62,15 +62,15 @@ interface LLMProvider { const llmProviders: Record = { ollama: { - name: "Ollama (Local Server)", - // baseUrl: "http://localhost:11434/v1/chat/completions", + name: "Ollama (NeuroJSON Server)", baseUrl: "", models: [ - { id: "qwen3-coder-next:latest", name: "Qwen 3 Coder Next" }, - { id: "qwen3-coder-careful:latest", name: "Qwen 3 Coder Careful" }, - { id: "qwen3.5:9b", name: "Qwen 3.5 9B" }, - { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (7.6B)" }, - { id: "qwen2.5-coder:7b", name: "Qwen 2.5 Coder 7B" }, + { id: "qwen3.6:27b", name: "Qwen 3.6 27B" }, + // { id: "qwen3-coder-next:latest", name: "Qwen 3 Coder Next" }, + // { id: "qwen3-coder-careful:latest", name: "Qwen 3 Coder Careful" }, + // { id: "qwen3.5:9b", name: "Qwen 3.5 9B" }, + // { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (7.6B)" }, + // { id: "qwen2.5-coder:7b", name: "Qwen 2.5 Coder 7B" }, ], noApiKey: true, }, @@ -1449,6 +1449,13 @@ const LLMPanel: React.FC = ({ + {provider === "ollama" && ( + + Using qwen3.6:27b on NeuroJSON server + + )} + + {provider !== "ollama" && ( Model + )} {provider === "local-ollama" && ( Date: Fri, 29 May 2026 12:47:30 -0400 Subject: [PATCH 60/61] fix(ollama): increase public endpoint rate limit to 20 requests/day --- backend/src/routes/ollama.public.routes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/routes/ollama.public.routes.js b/backend/src/routes/ollama.public.routes.js index b8981a0..c395e51 100644 --- a/backend/src/routes/ollama.public.routes.js +++ b/backend/src/routes/ollama.public.routes.js @@ -5,10 +5,10 @@ const { proxyChat, getTags } = require("../controllers/ollama.controller"); const dailyLimit = rateLimit({ windowMs: 24 * 60 * 60 * 1000, // 24 hours - max: 10, + max: 20, standardHeaders: true, legacyHeaders: false, - message: { error: "Daily request limit reached. You can send up to 10 requests per day from this IP." }, + message: { error: "Daily request limit reached. You can send up to 20 requests per day from this IP." }, }); router.post("/chat", dailyLimit, proxyChat); From 60cfdb0e5050fe33aa20d0ab43de8ddc724884c9 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 1 Jun 2026 10:42:02 -0400 Subject: [PATCH 61/61] fix(preview): make 2D chart responsive to container width --- package.json | 4 +- src/utils/preview.js | 22 ++++- yarn.lock | 215 ++++++++++++++++++++++--------------------- 3 files changed, 135 insertions(+), 106 deletions(-) diff --git a/package.json b/package.json index 6f414ef..aac37d2 100644 --- a/package.json +++ b/package.json @@ -50,13 +50,13 @@ "react-redux": "^8.1.2", "react-router-dom": "^6.15.0", "react-scripts": "^5.0.1", - "react-syntax-highlighter": "^15.6.1", + "react-syntax-highlighter": "16.1.1", "sharp": "^0.33.5", "stats-js": "^1.0.1", "stats.js": "0.17.0", "three": "0.145.0", "typescript": "^5.1.6", - "uplot": "1.6.17", + "uplot": "1.6.32", "web-vitals": "^2.1.0", "xlsx": "^0.18.5" }, diff --git a/src/utils/preview.js b/src/utils/preview.js index 7e4dd6d..231a85e 100644 --- a/src/utils/preview.js +++ b/src/utils/preview.js @@ -34,6 +34,7 @@ var lastvolumedata = null; var lastvolumedim = []; var lastclim = 0; var uplotInstance = null; +var uplotResizeObserver = null; var reqid = undefined; var canvas = null; @@ -135,6 +136,10 @@ function destroyPreview() { lastvolumedata = null; texture = undefined; + if (uplotResizeObserver !== null) { + uplotResizeObserver.disconnect(); + uplotResizeObserver = null; + } if (uplotInstance !== null) { uplotInstance.destroy(); uplotInstance = null; @@ -384,7 +389,7 @@ function dopreview(key, idx, isinternal, hastime) { // "Preview for " + // (isinternal ? intdata[idx][3] : window.extdata[idx][3]), "Data Preview", - width: 1100, + width: Math.max(300, $("#chartpanel").width() - 24), height: 400, series: [{}, {}], axes: [ @@ -507,6 +512,21 @@ function dopreview(key, idx, isinternal, hastime) { // }); } + if (uplotResizeObserver !== null) { + uplotResizeObserver.disconnect(); + } + uplotResizeObserver = new ResizeObserver(() => { + requestAnimationFrame(() => { + if (uplotInstance) { + uplotInstance.setSize({ + width: Math.max(300, $("#chartpanel").width() - 24), + height: 400, + }); + } + }); + }); + uplotResizeObserver.observe(document.getElementById("chartpanel")); + // for spinner // --- Signal React that 2D preview is ready --- window.__previewType = "2d"; diff --git a/yarn.lock b/yarn.lock index e881a21..ec84ee7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1122,7 +1122,7 @@ "@babel/plugin-transform-modules-commonjs" "^7.27.1" "@babel/plugin-transform-typescript" "^7.27.1" -"@babel/runtime@7.26.10", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.20.13", "@babel/runtime@^7.23.9", "@babel/runtime@^7.3.1", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": +"@babel/runtime@7.26.10", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.16.3", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.20.13", "@babel/runtime@^7.23.9", "@babel/runtime@^7.28.4", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": version "7.26.10" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.10.tgz#a07b4d8fa27af131a633d7b3524db803eb4764c2" integrity sha512-2WJMeRQPHKSPemqk/awGrAiuFfzBmOIPXKizAsVhWH9YJqLZ0H+HS4c8loHGgW6utJ3E/ejXQUsiGaQy2NZ9Fw== @@ -2895,12 +2895,12 @@ dependencies: "@types/node" "*" -"@types/hast@^2.0.0": - version "2.3.10" - resolved "https://registry.yarnpkg.com/@types/hast/-/hast-2.3.10.tgz#5c9d9e0b304bbb8879b857225c5ebab2d81d7643" - integrity sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw== +"@types/hast@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/hast/-/hast-3.0.4.tgz#1d6b39993b82cea6ad783945b0508c25903e15aa" + integrity sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ== dependencies: - "@types/unist" "^2" + "@types/unist" "*" "@types/hoist-non-react-statics@^3.3.1": version "3.3.6" @@ -3030,6 +3030,11 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f" integrity sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA== +"@types/prismjs@^1.0.0": + version "1.26.6" + resolved "https://registry.yarnpkg.com/@types/prismjs/-/prismjs-1.26.6.tgz#6ea27c126d645319ae4f7055eda63a9e835c0187" + integrity sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw== + "@types/prop-types@*", "@types/prop-types@^15.7.12": version "15.7.15" resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.15.tgz#e6e5a86d602beaca71ce5163fadf5f95d70931c7" @@ -3158,7 +3163,12 @@ resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.7.tgz#baccb07a970b91707df3a3e8ba6896c57ead2d11" integrity sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw== -"@types/unist@^2": +"@types/unist@*": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/unist/-/unist-3.0.3.tgz#acaab0f919ce69cce629c2d4ed2eb4adc1b6c20c" + integrity sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q== + +"@types/unist@^2.0.0": version "2.0.11" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.11.tgz#11af57b127e32487774841f7a4e54eab166d03c4" integrity sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA== @@ -4435,20 +4445,20 @@ char-regex@^2.0.0: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-2.0.2.tgz#81385bb071af4df774bff8721d0ca15ef29ea0bb" integrity sha512-cbGOjAptfM2LVmWhwRFHEKTPkLwNddVmuqYZQt895yXwAsWsXObCG+YN4DGQ/JBtT4GP1a1lPPdio2z413LmTg== -character-entities-legacy@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz#94bc1845dce70a5bb9d2ecc748725661293d8fc1" - integrity sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA== +character-entities-legacy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz#76bc83a90738901d7bc223a9e93759fdd560125b" + integrity sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ== -character-entities@^1.0.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.4.tgz#e12c3939b7eaf4e5b15e7ad4c5e28e1d48c5b16b" - integrity sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw== +character-entities@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-2.0.2.tgz#2d09c2e72cd9523076ccb21157dff66ad43fcc22" + integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== -character-reference-invalid@^1.0.0: - version "1.1.4" - resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" - integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== +character-reference-invalid@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz#85c66b041e43b47210faf401278abf808ac45cb9" + integrity sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw== check-types@^11.2.3: version "11.2.3" @@ -4641,10 +4651,10 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" -comma-separated-tokens@^1.0.0: - version "1.0.8" - resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" - integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== +comma-separated-tokens@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" + integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== commander@^11.1.0: version "11.1.0" @@ -5415,6 +5425,13 @@ decimal.js@^10.2.1: resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.5.0.tgz#0f371c7cf6c4898ce0afb09836db73cd82010f22" integrity sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw== +decode-named-character-reference@^1.0.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz#3e40603760874c2e5867691b599d73a7da25b53f" + integrity sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q== + dependencies: + character-entities "^2.0.0" + decode-uri-component@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.4.1.tgz#2ac4859663c704be22bf7db760a1494a49ab2cc5" @@ -7205,21 +7222,23 @@ hasown@^2.0.2: dependencies: function-bind "^1.1.2" -hast-util-parse-selector@^2.0.0: - version "2.2.5" - resolved "https://registry.yarnpkg.com/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz#d57c23f4da16ae3c63b3b6ca4616683313499c3a" - integrity sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ== +hast-util-parse-selector@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz#352879fa86e25616036037dd8931fb5f34cb4a27" + integrity sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A== + dependencies: + "@types/hast" "^3.0.0" -hastscript@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-6.0.0.tgz#e8768d7eac56c3fdeac8a92830d58e811e5bf640" - integrity sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w== +hastscript@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-9.0.1.tgz#dbc84bef6051d40084342c229c451cd9dc567dff" + integrity sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w== dependencies: - "@types/hast" "^2.0.0" - comma-separated-tokens "^1.0.0" - hast-util-parse-selector "^2.0.0" - property-information "^5.0.0" - space-separated-tokens "^1.0.0" + "@types/hast" "^3.0.0" + comma-separated-tokens "^2.0.0" + hast-util-parse-selector "^4.0.0" + property-information "^7.0.0" + space-separated-tokens "^2.0.0" he@^1.2.0: version "1.2.0" @@ -7521,18 +7540,18 @@ ipaddr.js@^2.0.1: resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.2.0.tgz#d33fa7bac284f4de7af949638c9d68157c6b92e8" integrity sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA== -is-alphabetical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d" - integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== +is-alphabetical@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-2.0.1.tgz#01072053ea7c1036df3c7d19a6daaec7f19e789b" + integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ== -is-alphanumerical@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz#7eb9a2431f855f6b1ef1a78e326df515696c4dbf" - integrity sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A== +is-alphanumerical@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz#7c03fbe96e3e931113e57f964b0a368cc2dfd875" + integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw== dependencies: - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" + is-alphabetical "^2.0.0" + is-decimal "^2.0.0" is-arguments@^1.1.1: version "1.2.0" @@ -7633,10 +7652,10 @@ is-date-object@^1.0.5, is-date-object@^1.1.0: call-bound "^1.0.2" has-tostringtag "^1.0.2" -is-decimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.4.tgz#65a3a5958a1c5b63a706e1b333d7cd9f630d3fa5" - integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw== +is-decimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-2.0.1.tgz#9469d2dc190d0214fd87d78b78caecc0cc14eef7" + integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A== is-docker@^2.0.0, is-docker@^2.1.1: version "2.2.1" @@ -7689,10 +7708,10 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-hexadecimal@^1.0.0: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz#cc35c97588da4bd49a8eedd6bc4082d44dcb23a7" - integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== +is-hexadecimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz#86b5bf668fca307498d319dfc03289d781a90027" + integrity sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg== is-map@^2.0.2, is-map@^2.0.3: version "2.0.3" @@ -9713,17 +9732,18 @@ parent-module@^2.0.0: dependencies: callsites "^3.1.0" -parse-entities@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-2.0.0.tgz#53c6eb5b9314a1f4ec99fa0fdf7ce01ecda0cbe8" - integrity sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ== +parse-entities@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-4.0.2.tgz#61d46f5ed28e4ee62e9ddc43d6b010188443f159" + integrity sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw== dependencies: - character-entities "^1.0.0" - character-entities-legacy "^1.0.0" - character-reference-invalid "^1.0.0" - is-alphanumerical "^1.0.0" - is-decimal "^1.0.0" - is-hexadecimal "^1.0.0" + "@types/unist" "^2.0.0" + character-entities-legacy "^3.0.0" + character-reference-invalid "^2.0.0" + decode-named-character-reference "^1.0.0" + is-alphanumerical "^2.0.0" + is-decimal "^2.0.0" + is-hexadecimal "^2.0.0" parse-json@^5.0.0, parse-json@^5.2.0: version "5.2.0" @@ -10518,16 +10538,11 @@ pretty-format@^29.0.0, pretty-format@^29.7.0: ansi-styles "^5.0.0" react-is "^18.0.0" -prismjs@^1.27.0: +prismjs@^1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.30.0.tgz#d9709969d9d4e16403f6f348c63553b19f0975a9" integrity sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw== -prismjs@~1.27.0: - version "1.27.0" - resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057" - integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== - process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" @@ -10564,12 +10579,10 @@ prop-types@^15.6.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" -property-information@^5.0.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/property-information/-/property-information-5.6.0.tgz#61675545fb23002f245c6540ec46077d4da3ed69" - integrity sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA== - dependencies: - xtend "^4.0.0" +property-information@^7.0.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/property-information/-/property-information-7.1.0.tgz#b622e8646e02b580205415586b40804d3e8bfd5d" + integrity sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ== proxy-addr@~2.0.7: version "2.0.7" @@ -10874,17 +10887,17 @@ react-scripts@^5.0.1: optionalDependencies: fsevents "^2.3.2" -react-syntax-highlighter@^15.6.1: - version "15.6.1" - resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-15.6.1.tgz#fa567cb0a9f96be7bbccf2c13a3c4b5657d9543e" - integrity sha512-OqJ2/vL7lEeV5zTJyG7kmARppUjiB9h9udl4qHQjjgEos66z00Ia0OckwYfRxCSFrW8RJIBnsBwQsHZbVPspqg== +react-syntax-highlighter@16.1.1: + version "16.1.1" + resolved "https://registry.yarnpkg.com/react-syntax-highlighter/-/react-syntax-highlighter-16.1.1.tgz#928459855d375f5cfc8e646071e20d541cebcb52" + integrity sha512-PjVawBGy80C6YbC5DDZJeUjBmC7skaoEUdvfFQediQHgCL7aKyVHe57SaJGfQsloGDac+gCpTfRdtxzWWKmCXA== dependencies: - "@babel/runtime" "^7.3.1" + "@babel/runtime" "^7.28.4" highlight.js "^10.4.1" highlightjs-vue "^1.0.0" lowlight "^1.17.0" - prismjs "^1.27.0" - refractor "^3.6.0" + prismjs "^1.30.0" + refractor "^5.0.0" react-textarea-autosize@^8.3.2: version "8.5.9" @@ -11009,14 +11022,15 @@ reflect.getprototypeof@^1.0.6, reflect.getprototypeof@^1.0.9: get-proto "^1.0.1" which-builtin-type "^1.2.1" -refractor@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/refractor/-/refractor-3.6.0.tgz#ac318f5a0715ead790fcfb0c71f4dd83d977935a" - integrity sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA== +refractor@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/refractor/-/refractor-5.0.0.tgz#85daf0448a6d947f5361796eb22c31733d61d904" + integrity sha512-QXOrHQF5jOpjjLfiNk5GFnWhRXvxjUVnlFxkeDmewR5sXkr3iM46Zo+CnRR8B+MDVqkULW4EcLVcRBNOPXHosw== dependencies: - hastscript "^6.0.0" - parse-entities "^2.0.0" - prismjs "~1.27.0" + "@types/hast" "^3.0.0" + "@types/prismjs" "^1.0.0" + hastscript "^9.0.0" + parse-entities "^4.0.0" regenerate-unicode-properties@^10.2.0: version "10.2.0" @@ -11707,10 +11721,10 @@ sourcemap-codec@^1.4.8: resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== -space-separated-tokens@^1.0.0: - version "1.1.5" - resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899" - integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== +space-separated-tokens@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz#1ecd9d2350a3844572c3f4a312bceb018348859f" + integrity sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q== spdy-transport@^3.0.0: version "3.0.0" @@ -12724,10 +12738,10 @@ update-browserslist-db@^1.1.3: escalade "^3.2.0" picocolors "^1.1.1" -uplot@1.6.17: - version "1.6.17" - resolved "https://registry.yarnpkg.com/uplot/-/uplot-1.6.17.tgz#1f8fc07a0e48008798beca463523621ad66dcc46" - integrity sha512-WHNHvDCXURn+Qwb3QUUzP6rOxx+3kUZUspREyhkqmXCxFIND99l5z9intTh+uPEt+/EEu7lCaMjSd1uTfuTXfg== +uplot@1.6.32: + version "1.6.32" + resolved "https://registry.yarnpkg.com/uplot/-/uplot-1.6.32.tgz#c800a63b432bad692d6d746f44f0882aa73a49ae" + integrity sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw== uri-js@^4.2.2: version "4.4.1" @@ -13451,11 +13465,6 @@ xmlchars@^2.2.0: resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== -xtend@^4.0.0: - version "4.0.2" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" - integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== - xtend@~2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-2.1.2.tgz#6efecc2a4dad8e6962c4901b337ce7ba87b5d28b"