diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 676c0256d..fd49c87e6 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -150,6 +150,23 @@ function cocoWithRle(trackId: number, categoryName = 'fish') { }); } +function cocoWithHierarchy(trackId: number, child: string, parent: string) { + return { + images: [{ id: 1, file_name: 'frame_000001.jpg', frame_index: 0 }], + annotations: [{ + id: trackId, + image_id: 1, + category_id: 2, + bbox: [10, 20, 30, 40], + track_id: trackId, + }], + categories: [ + { id: 1, name: parent }, + { id: 2, name: child, supercategory: parent }, + ], + }; +} + // Below sets up data in the mockfs type testPairs = [string[], MultiTrackRecord, Record]; /* Viame.spec.json is an array in the format [CSV row Array, MultiTrackRecord, Attributes Object][] @@ -866,6 +883,59 @@ beforeEach(() => { }); describe('native.common', () => { + it('imports COCO annotations and applies valid producer hierarchy separately', async () => { + const imported = '/home/user/output/hierarchy.coco.json'; + await fs.writeJSON(imported, cocoWithHierarchy(41, 'shark', 'fish')); + + const result = await common.dataFileImport(settings, 'projectid1', imported); + + expect(result.warnings).toEqual([]); + expect((await common.loadConfig(settings, 'projectid1', urlMapper)).typeHierarchy) + .toEqual({ shark: 'fish' }); + expect((await common.loadDetections(settings, 'projectid1')).tracks[41].confidencePairs) + .toEqual([['shark', 1]]); + }); + + it('warns and skips a conflicting COCO hierarchy without dropping annotations', async () => { + const imported = '/home/user/output/conflict.coco.json'; + await common.saveConfig(settings, 'projectid1', { + typeHierarchy: { shark: 'animal' }, + }); + await fs.writeJSON(imported, cocoWithHierarchy(42, 'shark', 'fish')); + + const result = await common.dataFileImport(settings, 'projectid1', imported); + + expect(result.warnings).toEqual([ + 'The category hierarchy in the COCO file could not be applied: conflicting parents for ' + + '"shark": "animal" and "fish". Annotations were imported without changing the dataset ' + + 'type hierarchy.', + ]); + expect((await common.loadConfig(settings, 'projectid1', urlMapper)).typeHierarchy) + .toEqual({ shark: 'animal' }); + expect((await common.loadDetections(settings, 'projectid1')).tracks[42]).toBeDefined(); + }); + + it('promotes the first multicamera COCO hierarchy and warns on later conflicts', async () => { + const left = '/home/user/output/left-hierarchy.coco.json'; + const right = '/home/user/output/right-hierarchy.coco.json'; + await fs.writeJSON(left, cocoWithHierarchy(51, 'shark', 'fish')); + await fs.writeJSON(right, cocoWithHierarchy(52, 'shark', 'animal')); + + const result = await common.ingestDataFiles( + settings, + 'stereoDataset', + [], + { left, right }, + ); + + expect(result.meta.typeHierarchy).toEqual({ shark: 'fish' }); + expect(result.warnings).toEqual([ + 'The category hierarchy in the COCO file could not be applied: conflicting parents for ' + + '"shark": "fish" and "animal". Annotations were imported without changing the dataset ' + + 'type hierarchy.', + ]); + }); + it('preserves warnings from primary COCO files in input order', async () => { const first = '/home/user/output/first.coco.json'; const second = '/home/user/output/second.coco.json'; @@ -1336,6 +1406,70 @@ describe('native.common', () => { expect(await fs.pathExists(output)).toBe(false); }); + it('uses the parent hierarchy when exporting COCO from a selected multicam camera', async () => { + const parentId = 'coco-export-parent'; + const parentDir = common.getProjectDir(settings, parentId); + const cameraDir = common.getProjectDir(settings, `${parentId}/left`); + const seed = await common.loadJsonConfig( + common.getProjectDir(settings, 'projectid1').datasetFileAbsPath, + ); + await fs.ensureDir(cameraDir.basePath); + await fs.writeJSON(parentDir.datasetFileAbsPath, { + ...seed, + id: parentId, + typeHierarchy: { shark: 'fish' }, + }); + await fs.writeJSON(cameraDir.datasetFileAbsPath, { + ...seed, + id: `${parentId}/left`, + typeHierarchy: { shark: 'animal' }, + }); + await fs.writeJSON(npath.join(cameraDir.basePath, 'result.json'), { + version: AnnotationsCurrentVersion, + groups: {}, + tracks: { + 1: { + id: 1, + begin: 0, + end: 0, + attributes: {}, + confidencePairs: [['shark', 0.9]], + features: [{ frame: 0, bounds: [0, 0, 1, 1] }], + }, + }, + }); + + const output = '/home/user/output/selected-camera.coco.json'; + await common.exportDataset(settings, { + id: `${parentId}/left`, + path: output, + type: 'coco', + exclude: false, + typeFilter: new Set(), + }); + + const exported = await fs.readJSON(output); + expect(exported.categories).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: 'shark', supercategory: 'fish' }), + ])); + + const corruptParent = await fs.readJSON(parentDir.datasetFileAbsPath); + corruptParent.typeHierarchy = { fish: 'fish' }; + await fs.writeJSON(parentDir.datasetFileAbsPath, corruptParent); + await fs.remove(output); + await expect(common.exportDataset(settings, { + id: `${parentId}/left`, + path: output, + type: 'coco', + exclude: false, + typeFilter: new Set(), + })).rejects.toThrow( + 'Type hierarchy is invalid: self edge "fish -> fish". ' + + 'No COCO file was exported.', + ); + expect(await fs.pathExists(output)).toBe(false); + }); + it('loadJsonConfig parses per-camera frame timestamps for multicam datasets', async () => { const data = await common.loadConfig(settings, 'stereoDataset', urlMapper); expect(data.multiCamMedia).not.toBeNull(); @@ -1476,6 +1610,79 @@ describe('native.common', () => { expect(rightMeta.metadataOriginalName).toBeUndefined(); }); + it('promotes the first camera COCO hierarchy and reports a later conflict', async () => { + const leftTrackFile = '/home/user/output/finalize-left.coco.json'; + const rightTrackFile = '/home/user/output/finalize-right.coco.json'; + await fs.writeJSON(leftTrackFile, cocoWithHierarchy(61, 'shark', 'fish')); + await fs.writeJSON(rightTrackFile, cocoWithHierarchy(62, 'shark', 'animal')); + const payload = await beginMultiCamImport({ + datasetName: 'hierarchy_multicam', + defaultDisplay: 'left', + sourceList: { + left: { + sourcePath: '/home/user/data/imageSuccess', + trackFile: leftTrackFile, + }, + right: { + sourcePath: '/home/user/data/imageSuccess', + trackFile: rightTrackFile, + }, + }, + type: 'image-sequence', + }); + + const result = await common.finalizeMediaImport(settings, payload); + + expect(result.meta.typeHierarchy).toEqual({ shark: 'fish' }); + expect(result.importWarnings).toContain( + 'Camera "right" type hierarchy was skipped: conflicting parents for "shark": "fish" ' + + 'and "animal"', + ); + const project = common.getProjectDir(settings, result.meta.id); + const leftMeta = await fs.readJSON(npath.join(project.basePath, 'left', 'dataset.json')); + const rightMeta = await fs.readJSON(npath.join(project.basePath, 'right', 'dataset.json')); + expect(leftMeta.typeHierarchy).toBeUndefined(); + expect(rightMeta.typeHierarchy).toBeUndefined(); + }); + + it('promotes hierarchies in stored camera order and still imports unlisted cameras', async () => { + const leftTrackFile = '/home/user/output/ordered-left.coco.json'; + const rightTrackFile = '/home/user/output/ordered-right.coco.json'; + await fs.writeJSON(leftTrackFile, cocoWithHierarchy(63, 'shark', 'fish')); + await fs.writeJSON(rightTrackFile, cocoWithHierarchy(64, 'shark', 'animal')); + const payload = await beginMultiCamImport({ + datasetName: 'ordered_hierarchy_multicam', + defaultDisplay: 'left', + sourceList: { + left: { + sourcePath: '/home/user/data/imageSuccess', + trackFile: leftTrackFile, + }, + right: { + sourcePath: '/home/user/data/imageSuccess', + trackFile: rightTrackFile, + }, + }, + type: 'image-sequence', + }); + // Simulate a legacy/partial stored order: unknown names are ignored and cameras omitted + // from the list are appended after the explicitly ordered cameras. + if (payload.jsonConfig.multiCam) { + payload.jsonConfig.multiCam.cameraOrder = ['missing', 'right']; + } + + const result = await common.finalizeMediaImport(settings, payload); + + expect(result.meta.typeHierarchy).toEqual({ shark: 'animal' }); + expect(result.importWarnings).toContain( + 'Camera "left" type hierarchy was skipped: conflicting parents for "shark": "animal" ' + + 'and "fish"', + ); + const project = common.getProjectDir(settings, result.meta.id); + expect(await fs.pathExists(npath.join(project.basePath, 'right', 'dataset.json'))).toBe(true); + expect(await fs.pathExists(npath.join(project.basePath, 'left', 'dataset.json'))).toBe(true); + }); + it('warns instead of refusing when a folder holds two reserved metadata attachments', async () => { // The dialog's "Metadata File (Optional)" picker is the only place the user can settle // this, and it opens only once beginMediaImport returns, so the import must survive. diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 7be5f49c0..e1c8ae6e4 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -101,10 +101,26 @@ const invalidHierarchyMessage = (reason: string) => ( `Type hierarchy is invalid: ${reason}. No configuration was changed.` ); -const corruptHierarchyExportMessage = (reason: string) => ( - `Type hierarchy is invalid: ${reason}. No configuration file was exported.` +const corruptHierarchyExportMessage = (reason: string, artifact = 'configuration file') => ( + `Type hierarchy is invalid: ${reason}. No ${artifact} was exported.` ); +function normalizedHierarchyForExport( + meta: { typeHierarchy?: unknown }, + artifact?: string, +) { + try { + return Object.prototype.hasOwnProperty.call(meta, 'typeHierarchy') + ? normalizeTypeHierarchy(meta.typeHierarchy) + : undefined; + } catch (error) { + if (error instanceof TypeHierarchyError) { + throw new Error(corruptHierarchyExportMessage(error.reason, artifact)); + } + throw error; + } +} + class DataFileJsonParseError extends Error {} /** @@ -1374,7 +1390,7 @@ async function _ingestFilePath( (DatasetConfigMutable & { fps?: number }), string[], boolean, string, ] | null> { const { - datasetId, path, additive, additivePrepend, configMeta, + datasetId, path, additive, additivePrepend, configMeta, cocoHierarchy, } = plan; if (!fs.existsSync(path)) { return null; @@ -1411,6 +1427,9 @@ async function _ingestFilePath( const [parsedAnnotations, parsedMeta, cocoWarnings] = await coco.parseFile(path); annotations = parsedAnnotations; merge(meta, parsedMeta); + if (cocoHierarchy) { + meta.typeHierarchy = { ...cocoHierarchy }; + } warnings = warnings.concat(cocoWarnings); } else { // Regular dive json @@ -1482,6 +1501,8 @@ interface IngestFilePlan { additive: boolean; additivePrepend: string; configMeta?: StagedConfigImport; + cocoHierarchy?: Record; + configWarnings?: string[]; } async function loadCanonicalHierarchy(settings: Settings, datasetId: string): Promise { @@ -1614,6 +1635,30 @@ async function preflightIngestFiles( } throw error; } + } else if (jsonObject !== undefined && coco.isCocoJson(jsonObject)) { + const { hierarchy, warnings } = coco.typeHierarchyFromCategories(jsonObject); + entry.configWarnings = warnings; + if (hierarchy !== undefined) { + try { + const write = resolveTypeHierarchy( + hierarchyCandidate ?? null, + true, + hierarchy, + 'additive', + ); + if (write.action === 'set') { + hierarchyCandidate = write.hierarchy; + entry.cocoHierarchy = { ...write.hierarchy }; + } + } catch (error) { + if (!(error instanceof TypeHierarchyError)) { + throw error; + } + entry.configWarnings = entry.configWarnings.concat( + coco.invalidCocoHierarchyMessage(error.reason), + ); + } + } } } } @@ -1669,7 +1714,7 @@ async function ingestDataFiles( ); if (results !== null) { const [newMeta, warnings, metadataConfig, auxiliaryPath] = results; - outwarnings = outwarnings.concat(warnings); + outwarnings = outwarnings.concat(warnings, entry.configWarnings || []); mergeStagedImportedConfig(meta, newMeta, additive); if (metadataConfig) { importedConfigCopies.push(auxiliaryPath); @@ -2267,12 +2312,34 @@ async function dataFileImport(settings: Settings, id: string, path: string, addi return result; } +function mergeCameraTypeHierarchy( + promoted: Record | undefined, + cameraHierarchy: Record, + cameraName: string, +): { hierarchy: Record; warning?: string } { + try { + const write = resolveTypeHierarchy(promoted ?? null, true, cameraHierarchy, 'additive'); + return { + hierarchy: write.action === 'set' ? { ...write.hierarchy } : { ...(promoted || {}) }, + }; + } catch (error) { + if (!(error instanceof TypeHierarchyError)) { + throw error; + } + return { + hierarchy: { ...(promoted || {}) }, + warning: `Camera "${cameraName}" type hierarchy was skipped: ${error.reason}`, + }; + } +} + async function _importTrackFile( settings: Settings, dsId: string, projectDirAbsPath: string, jsonConfig: JsonConfig, userTrackFileAbsPath: string, + promoteTypeHierarchy = false, ) { /* custom image sort */ if (jsonConfig.imageListPath === undefined) { @@ -2281,9 +2348,17 @@ async function _importTrackFile( if (jsonConfig.transcodedImageFiles) { jsonConfig.transcodedImageFiles.sort(strNumericCompare); } + let promotedTypeHierarchy: Record | undefined; + let warnings: string[] = []; if (userTrackFileAbsPath) { const processed = await ingestDataFiles(settings, dsId, [userTrackFileAbsPath], undefined, validImageNamesMap(jsonConfig)); - merge(jsonConfig, processed.meta); + const importedMeta = { ...processed.meta }; + if (promoteTypeHierarchy && importedMeta.typeHierarchy) { + promotedTypeHierarchy = importedMeta.typeHierarchy; + delete importedMeta.typeHierarchy; + } + merge(jsonConfig, importedMeta); + warnings = processed.warnings; if (processed.processedFiles.length === 0) { await _saveSerialized(settings, dsId, dive.makeEmptyAnnotationFile(), true); } @@ -2291,7 +2366,7 @@ async function _importTrackFile( await _saveSerialized(settings, dsId, dive.makeEmptyAnnotationFile(), true); } await saveProjectConfig(projectDirAbsPath, jsonConfig); - return jsonConfig; + return { jsonConfig, typeHierarchy: promotedTypeHierarchy, warnings }; } /** @@ -2446,11 +2521,22 @@ async function finalizeMediaImport( } //We need to create datasets for each of the multiCam folders as well + let promotedTypeHierarchy = jsonConfig.typeHierarchy + ? { ...jsonConfig.typeHierarchy } + : undefined; + const importWarnings: string[] = []; if (datasetType === MultiType && jsonConfig.multiCam?.cameras) { - const cameraNameAndData = Object.entries(jsonConfig.multiCam.cameras); + const { cameras } = jsonConfig.multiCam; + const orderedCameraNames = orderedMultiCamCameraNames(jsonConfig.multiCam); + const cameraNames = [ + ...orderedCameraNames, + ...Object.keys(cameras).filter((name) => !orderedCameraNames.includes(name)), + ]; + const cameraNameAndData = cameraNames.map( + (cameraName) => [cameraName, cameras[cameraName]] as const, + ); for (let i = 0; i < cameraNameAndData.length; i += 1) { - const cameraName = cameraNameAndData[i][0]; - const cameraData = cameraNameAndData[i][1]; + const [cameraName, cameraData] = cameraNameAndData[i]; const jsonClone = { ...cloneDeep(jsonConfig), ...cameraData }; if (!cameraData.metadataFile) { @@ -2462,6 +2548,7 @@ async function finalizeMediaImport( jsonClone.transcodedVideoFile = cameraData.transcodedVideoFile || ''; jsonClone.transcodedImageFiles = cameraData.transcodedImageFiles || []; jsonClone.subType = null; + delete jsonClone.typeHierarchy; // eslint-disable-next-line no-await-in-loop const cameraDirAbsPath = await _initializeProjectDir(settings, jsonClone); let multiCamTrackFile = ''; @@ -2469,10 +2556,40 @@ async function finalizeMediaImport( multiCamTrackFile = args.multiCamTrackFiles[cameraName]; } // eslint-disable-next-line no-await-in-loop - await _importTrackFile(settings, jsonClone.id, cameraDirAbsPath, jsonClone, multiCamTrackFile); + const imported = await _importTrackFile( + settings, + jsonClone.id, + cameraDirAbsPath, + jsonClone, + multiCamTrackFile, + true, + ); + importWarnings.push(...imported.warnings); + if (imported.typeHierarchy) { + const merged = mergeCameraTypeHierarchy( + promotedTypeHierarchy, + imported.typeHierarchy, + cameraName, + ); + promotedTypeHierarchy = merged.hierarchy; + if (merged.warning) { + importWarnings.push(merged.warning); + } + } } } - const finalJsonConfig = await _importTrackFile(settings, jsonConfig.id, projectDirAbsPath, jsonConfig, args.trackFileAbsPath); + if (promotedTypeHierarchy && Object.keys(promotedTypeHierarchy).length > 0) { + jsonConfig.typeHierarchy = promotedTypeHierarchy; + } + const finalImport = await _importTrackFile( + settings, + jsonConfig.id, + projectDirAbsPath, + jsonConfig, + args.trackFileAbsPath, + ); + const finalJsonConfig = finalImport.jsonConfig; + importWarnings.push(...finalImport.warnings); if (args.configFileAbsPath) { await dataFileImport(settings, jsonConfig.id, args.configFileAbsPath); } @@ -2480,6 +2597,7 @@ async function finalizeMediaImport( type: JobType.Conversion, meta: finalJsonConfig, mediaList: srcDstList, + importWarnings, }; return conversionJobArgs; } @@ -2588,6 +2706,15 @@ async function exportDataset(settings: Settings, args: ExportDatasetArgs) { const projectDirInfo = await getValidatedProjectDir(settings, args.id); const meta = await loadJsonConfig(projectDirInfo.datasetFileAbsPath); const data = await loadAnnotationFile(projectDirInfo.trackFileAbsPath); + const { cameraName } = parseCompositeDatasetId(args.id); + if (cameraName) { + const hierarchy = await loadCanonicalHierarchy(settings, args.id); + if (hierarchy === null) { + delete meta.typeHierarchy; + } else { + meta.typeHierarchy = hierarchy as Record; + } + } if (args.type === 'json') { return dive.serializeFile(args.path, data, meta, args.typeFilter, { excludeBelowThreshold: args.exclude, @@ -2595,6 +2722,12 @@ async function exportDataset(settings: Settings, args: ExportDatasetArgs) { }); } if (args.type === 'coco') { + const hierarchy = normalizedHierarchyForExport(meta, 'COCO file'); + if (hierarchy) { + meta.typeHierarchy = { ...hierarchy }; + } else { + delete meta.typeHierarchy; + } return coco.serializeFile(args.path, data, meta, args.typeFilter, { excludeBelowThreshold: args.exclude, }); @@ -2618,17 +2751,7 @@ async function exportConfiguration(settings: Settings, args: ExportConfiguration } } const output: DatasetConfigMutable & { version: number} = { version: meta.version }; - let hierarchy; - try { - hierarchy = Object.prototype.hasOwnProperty.call(meta, 'typeHierarchy') - ? normalizeTypeHierarchy(meta.typeHierarchy) - : undefined; - } catch (error) { - if (error instanceof TypeHierarchyError) { - throw new Error(corruptHierarchyExportMessage(error.reason)); - } - throw error; - } + const hierarchy = normalizedHierarchyForExport(meta); if (DatasetConfigMutableKeys.some((key) => key in meta)) { // DIVE Configuration File fields (attributes, styles, FPS, …) merge(output, pick(meta, DatasetConfigMutableKeys)); diff --git a/client/platform/desktop/backend/serializers/coco.spec.ts b/client/platform/desktop/backend/serializers/coco.spec.ts index 4f65263a0..df2d40b93 100644 --- a/client/platform/desktop/backend/serializers/coco.spec.ts +++ b/client/platform/desktop/backend/serializers/coco.spec.ts @@ -2,7 +2,20 @@ import fs from 'fs-extra'; import mockfs from 'mock-fs'; import { AnnotationSchema } from 'dive-common/apispec'; import { AnnotationsCurrentVersion, JsonConfig } from 'platform/desktop/constants'; -import { isCocoJson, parseFile, serializeFile } from 'platform/desktop/backend/serializers/coco'; +import { + CATEGORY_MISSING_NAME_WARNING, + DIVE_CONFIDENCE_PAIRS_INVALID_WARNING, + PROB_DUPLICATE_CATEGORY_WARNING, + PROB_LENGTH_MISMATCH_WARNING, + SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING, + SUPERCATEGORY_MULTI_PARENT_WARNING, + isCocoJson, + parseFile, + serializeFile, + typeHierarchyFromCategories, +} from 'platform/desktop/backend/serializers/coco'; + +const kwcocoProfile = fs.readJSONSync('../testutils/kwcoco/import-profile.json'); const cocoInput = { images: [{ id: 1, file_name: 'frame_000001.jpg', frame_index: 1 }], @@ -77,6 +90,47 @@ beforeEach(() => { }); describe('COCO serializer', () => { + it('matches the shared exact-vector and hierarchy import profile', async () => { + const profile = kwcocoProfile.highestFrameExact; + mockfs({ + '/input': { + 'profile.json': JSON.stringify(profile.document), + }, + }); + + const [parsed, , warnings] = await parseFile('/input/profile.json'); + expect(parsed.tracks[profile.trackId].confidencePairs).toEqual(profile.expectedPairs); + expect(typeHierarchyFromCategories(profile.document).hierarchy) + .toEqual(profile.expectedHierarchy); + expect(warnings).toEqual([]); + }); + + it('matches server ordering when external images omit frame indices', async () => { + const profile = kwcocoProfile.missingFrameIndexExact; + mockfs({ + '/input': { + 'profile.json': JSON.stringify(profile.document), + }, + }); + + const [parsed, , warnings] = await parseFile('/input/profile.json'); + expect(parsed.tracks[profile.trackId].confidencePairs).toEqual(profile.expectedPairs); + expect(warnings).toEqual([]); + }); + + it('rejects empty DIVE confidence pairs from the shared import profile', async () => { + const profile = kwcocoProfile.emptyDiveConfidencePairs; + mockfs({ + '/input': { + 'profile.json': JSON.stringify(profile.document), + }, + }); + + const [parsed, , warnings] = await parseFile('/input/profile.json'); + expect(parsed.tracks[profile.trackId].confidencePairs).toEqual(profile.expectedPairs); + expect(warnings).toEqual([DIVE_CONFIDENCE_PAIRS_INVALID_WARNING]); + }); + it('detects base coco shape', () => { expect(isCocoJson(cocoInput)).toBe(true); expect(isCocoJson({ images: [], annotations: [] })).toBe(false); @@ -205,6 +259,7 @@ describe('COCO serializer', () => { 'dive_detection_attributes', 'dive_track_attributes', 'dive_notes', + 'dive_confidence_pairs', ]); expect(out.annotations).toHaveLength(1); expect(out.annotations[0].dive_detection_attributes).toEqual({ visibility: 'poor' }); @@ -259,6 +314,340 @@ describe('COCO serializer', () => { const [, parsedMeta] = await parseFile('/input/coco.json'); expect(parsedMeta).not.toHaveProperty('datasetInfo'); }); + + it('imports a pruned KWCOCO probability vector by raw category position', async () => { + mockfs({ + '/input': { + 'prob.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [{ + id: 1, + image_id: 1, + category_id: 3, + bbox: [0, 0, 1, 1], + track_id: 9, + prob: [0.1, 0.8, 0.2], + }], + categories: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }, { id: 3, name: 'c' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/prob.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['b', 0.8], ['c', 0.2], ['a', 0.1]]); + expect(warnings).toEqual([]); + }); + + it('keeps unnamed category slots while mapping prob vectors', async () => { + mockfs({ + '/input': { + 'unnamed-prob.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [{ + id: 1, + image_id: 1, + category_id: 3, + bbox: [0, 0, 1, 1], + track_id: 9, + prob: [0.9, 0.8, 0.7], + }], + categories: [{ id: 1, name: 'a' }, { id: 2 }, { id: 3, name: 'c' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/unnamed-prob.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['a', 0.9], ['c', 0.7]]); + expect(warnings).toEqual([]); + }); + + it('clamps finite prob values and prunes to the strongest ten above epsilon', async () => { + const categories = Array.from({ length: 12 }, (_, id) => ({ id: id + 1, name: `type-${id}` })); + mockfs({ + '/input': { + 'pruned-prob.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [{ + id: 1, + image_id: 1, + category_id: 1, + bbox: [0, 0, 1, 1], + track_id: 9, + prob: [1.5, ...Array.from({ length: 10 }, (_, index) => 0.9 - (index * 0.01)), 0.0005], + }], + categories, + }), + }, + }); + const [parsed] = await parseFile('/input/pruned-prob.json'); + expect(parsed.tracks[9].confidencePairs).toHaveLength(10); + expect(parsed.tracks[9].confidencePairs[0]).toEqual(['type-0', 1]); + expect(parsed.tracks[9].confidencePairs.map(([name]) => name)).not.toContain('type-11'); + }); + + it('warns once for invalid external prob vectors and falls back to category score', async () => { + mockfs({ + '/input': { + 'bad-prob.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [ + { + id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 9, score: 0.4, prob: [0.8], + }, + { + id: 2, image_id: 1, category_id: 2, bbox: [0, 0, 1, 1], track_id: 10, score: 0.5, prob: [0.8], + }, + ], + categories: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/bad-prob.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['a', 0.4]]); + expect(parsed.tracks[10].confidencePairs).toEqual([['b', 0.5]]); + expect(warnings).toEqual([PROB_LENGTH_MISMATCH_WARNING]); + }); + + it('prefers valid exact DIVE confidence pairs, including zero values', async () => { + mockfs({ + '/input': { + 'exact-pairs.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [{ + id: 1, + image_id: 1, + category_id: 1, + bbox: [0, 0, 1, 1], + track_id: 9, + prob: [0.9, 0.1], + dive_confidence_pairs: [['sparse', 0], ['exact', 0.75]], + }], + categories: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/exact-pairs.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['sparse', 0], ['exact', 0.75]]); + expect(warnings).toEqual([]); + }); + + it('uses the highest frame classification regardless of annotation file order', async () => { + mockfs({ + '/input': { + 'highest-frame.json': JSON.stringify({ + images: [ + { id: 1, file_name: 'early.jpg', frame_index: 2 }, + { id: 2, file_name: 'late.jpg', frame_index: 9 }, + ], + annotations: [ + { + id: 1, image_id: 2, category_id: 1, bbox: [0, 0, 1, 1], track_id: 9, score: 0.9, + }, + { + id: 2, image_id: 1, category_id: 2, bbox: [0, 0, 1, 1], track_id: 9, score: 0.2, + }, + ], + categories: [{ id: 1, name: 'late' }, { id: 2, name: 'early' }], + }), + }, + }); + const [parsed] = await parseFile('/input/highest-frame.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['late', 0.9]]); + }); + + it('uses the greatest annotation id when classifications share the highest frame', async () => { + mockfs({ + '/input': { + 'same-frame.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 9 }], + annotations: [ + { + id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 9, score: 0.2, + }, + { + id: 2, image_id: 1, category_id: 2, bbox: [0, 0, 1, 1], track_id: 9, score: 0.8, + }, + ], + categories: [{ id: 1, name: 'first' }, { id: 2, name: 'last' }], + }), + }, + }); + const [parsed] = await parseFile('/input/same-frame.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['last', 0.8]]); + }); + + it('uses the same highest-frame classification after annotations are reordered', async () => { + const document = { + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 9 }], + annotations: [ + { + id: 42, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 9, score: 0.2, + }, + { + id: 99, image_id: 1, category_id: 2, bbox: [0, 0, 1, 1], track_id: 9, score: 0.8, + }, + ], + categories: [{ id: 1, name: 'lower-id' }, { id: 2, name: 'higher-id' }], + }; + await Promise.all([ + fs.writeJSON('/input/same-frame-order-a.json', document), + fs.writeJSON('/input/same-frame-order-b.json', { + ...document, + annotations: [...document.annotations].reverse(), + }), + ]); + const [[first], [second]] = await Promise.all([ + parseFile('/input/same-frame-order-a.json'), + parseFile('/input/same-frame-order-b.json'), + ]); + expect(first.tracks[9].confidencePairs).toEqual([['higher-id', 0.8]]); + expect(second.tracks[9].confidencePairs).toEqual([['higher-id', 0.8]]); + }); + + it('warns once for malformed DIVE confidence pairs and falls back to category scores', async () => { + mockfs({ + '/input': { + 'invalid-exact-pairs.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [ + { + id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 1, score: 0.1, dive_confidence_pairs: 'fish', + }, + { + id: 2, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 2, score: 0.2, dive_confidence_pairs: [['fish']], + }, + { + id: 3, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 3, score: 0.3, dive_confidence_pairs: [['fish', 0.1], ['fish', 0.2]], + }, + { + id: 4, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 4, score: 0.4, dive_confidence_pairs: [['fish', Number.POSITIVE_INFINITY]], + }, + { + id: 5, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 5, score: 0.5, dive_confidence_pairs: [['fish', 1.1]], + }, + ], + categories: [{ id: 1, name: 'fish' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/invalid-exact-pairs.json'); + expect(Object.values(parsed.tracks).map((track) => track.confidencePairs)) + .toEqual([['fish', 0.1], ['fish', 0.2], ['fish', 0.3], ['fish', 0.4], ['fish', 0.5]].map((pair) => [pair])); + expect(warnings).toEqual([DIVE_CONFIDENCE_PAIRS_INVALID_WARNING]); + }); + + it('extracts hierarchy edges and reports representational warnings deterministically', () => { + const result = typeHierarchyFromCategories({ + images: [], + annotations: [], + categories: [ + { id: 1, name: 'fish', supercategory: 'fish' }, + { + id: 2, name: 'shark', supercategory: 'fish', parents: ['fish', 'animal'], + }, + { id: 3 }, + ], + }); + expect(result.hierarchy).toEqual({ shark: 'fish' }); + expect(result.warnings).toEqual([ + SUPERCATEGORY_MULTI_PARENT_WARNING, + CATEGORY_MISSING_NAME_WARNING, + ]); + + expect(typeHierarchyFromCategories({ + images: [], annotations: [], categories: [{ id: 1, name: 'fish' }], + })).toEqual({ warnings: [] }); + + const duplicate = typeHierarchyFromCategories({ + images: [], annotations: [], categories: [{ id: 1, name: 'fish' }, { id: 2, name: 'fish' }], + }); + expect(duplicate.hierarchy).toBeUndefined(); + expect(duplicate.warnings).toEqual([SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING]); + }); + + it('warns once and ignores prob vectors when category names are duplicated', async () => { + mockfs({ + '/input': { + 'duplicate-prob.json': JSON.stringify({ + images: [{ id: 1, file_name: 'frame.jpg', frame_index: 0 }], + annotations: [{ + id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 1, 1], track_id: 9, prob: [0.2, 0.8], + }], + categories: [{ id: 1, name: 'fish' }, { id: 2, name: 'fish' }], + }), + }, + }); + const [parsed, , warnings] = await parseFile('/input/duplicate-prob.json'); + expect(parsed.tracks[9].confidencePairs).toEqual([['fish', 1]]); + expect(warnings).toEqual([PROB_DUPLICATE_CATEGORY_WARNING]); + }); + + it('matches the shared KWCOCO export and round-trip profile', async () => { + const profile = kwcocoProfile.exportRoundTrip; + const profileTrack = profile.tracks[0] as AnnotationSchema['tracks'][number]; + const source: AnnotationSchema = { + version: AnnotationsCurrentVersion, + groups: {}, + tracks: { + [profileTrack.id]: profileTrack, + }, + }; + const originalPairs = profileTrack.confidencePairs.map(([name, score]) => [name, score]); + await serializeFile('/output/kwcoco.json', source, { + ...imageMeta, + name: profile.datasetName, + originalImageFiles: [profile.imageFilenames['0']], + typeHierarchy: profile.typeHierarchy, + }); + const out = await fs.readJSON('/output/kwcoco.json'); + expect(out.categories.map(({ name }: { name: string }) => name)) + .toEqual(profile.expectedCategoryNames); + expect(Object.fromEntries(out.categories + .filter(({ supercategory }: { supercategory?: string }) => supercategory) + .map(({ name, supercategory }: { name: string; supercategory: string }) => ( + [name, supercategory] + )))).toEqual(profile.expectedParents); + expect(out.info.dive_extensions).toContain('dive_confidence_pairs'); + expect(out.annotations[0]).toMatchObject({ + category_id: 2, + track_id: profileTrack.id, + score: 0.75, + prob: profile.expectedProb, + dive_confidence_pairs: profile.expectedPairs, + }); + expect(source.tracks[profileTrack.id].confidencePairs).toEqual(originalPairs); + + await fs.writeJSON('/input/kwcoco.json', out); + const [parsed] = await parseFile('/input/kwcoco.json'); + expect(parsed.tracks[profileTrack.id].confidencePairs).toEqual(profile.expectedPairs); + }); + + it('filters the raw exported pair vector without mutating its source track', async () => { + const source: AnnotationSchema = { + version: AnnotationsCurrentVersion, + groups: {}, + tracks: { + 4: { + id: 4, + begin: 0, + end: 0, + confidencePairs: [['root', 0.2], ['leaf', 0.8]], + attributes: {}, + features: [{ frame: 0, bounds: [0, 0, 4, 4] }], + }, + }, + }; + await serializeFile('/output/filtered.json', source, { + ...imageMeta, + typeHierarchy: { leaf: 'root' }, + }, new Set(['leaf'])); + const out = await fs.readJSON('/output/filtered.json'); + expect(out.annotations[0].dive_confidence_pairs).toEqual([['leaf', 0.8]]); + expect(out.annotations[0].prob).toEqual([0.8, 0]); + expect(source.tracks[4].confidencePairs).toEqual([['root', 0.2], ['leaf', 0.8]]); + + await fs.writeJSON('/input/filtered.json', out); + const [parsed] = await parseFile('/input/filtered.json'); + expect(parsed.tracks[4].confidencePairs).toEqual([['leaf', 0.8]]); + }); }); afterEach(() => { diff --git a/client/platform/desktop/backend/serializers/coco.ts b/client/platform/desktop/backend/serializers/coco.ts index a02ff7c70..7d2c0ab4d 100644 --- a/client/platform/desktop/backend/serializers/coco.ts +++ b/client/platform/desktop/backend/serializers/coco.ts @@ -3,6 +3,7 @@ import { isEmpty } from 'lodash'; import { AnnotationSchema } from 'dive-common/apispec'; import { JsonConfig } from 'platform/desktop/constants'; import processTrackAttributes from 'platform/desktop/backend/native/attributeProcessor'; +import { strNumericCompare } from 'platform/desktop/sharedUtils'; import { TrackSupportedFeature } from 'vue-media-annotator/track'; type CocoImage = { @@ -13,8 +14,10 @@ type CocoImage = { type CocoCategory = { id: number; - name: string; + name?: string; keypoints?: string[]; + supercategory?: string | null; + parents?: unknown; }; const RLE_SEGMENTATION_WARNING = ( @@ -22,6 +25,91 @@ const RLE_SEGMENTATION_WARNING = ( + 'Bounding boxes and other annotation data were imported, but masks were skipped.' ); +const PROB_TOP_K = 10; +const PROB_EPSILON = 0.001; + +const PROB_LENGTH_MISMATCH_WARNING = ( + 'Some annotations had a "prob" array whose length did not match the number of categories. ' + + 'Class probabilities were ignored for those annotations; the primary category and score ' + + 'were imported instead.' +); +const PROB_DUPLICATE_CATEGORY_WARNING = ( + 'The COCO file contains duplicate category names, so "prob" arrays cannot be mapped to ' + + 'class names. Class probabilities were ignored; primary categories and scores were ' + + 'imported instead.' +); +const DIVE_CONFIDENCE_PAIRS_INVALID_WARNING = ( + 'Some annotations had malformed "dive_confidence_pairs" values. ' + + 'Those values were ignored; class probabilities or primary categories and scores were used instead.' +); +const SUPERCATEGORY_MULTI_PARENT_WARNING = ( + 'Some COCO categories declare multiple parents via "parents", which DIVE cannot ' + + 'represent. Only single-parent "supercategory" edges were imported.' +); +const SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING = ( + 'The COCO file contains duplicate category names, so category hierarchy edges cannot be ' + + 'mapped to class names. The dataset type hierarchy was left unchanged.' +); +const CATEGORY_MISSING_NAME_WARNING = ( + 'Some COCO categories have no non-empty string name. Those positional category slots were ' + + 'ignored when importing classifications and hierarchy edges.' +); +const SUPERCATEGORY_INVALID_WARNING = ( + 'The category hierarchy in the COCO file could not be applied: {reason}. ' + + 'Annotations were imported without changing the dataset type hierarchy.' +); + +function hasDuplicateCategoryNames(names: readonly (string | undefined)[]): boolean { + const named = names.filter((name): name is string => typeof name === 'string' && name.length > 0); + return new Set(named).size !== named.length; +} + +function invalidCocoHierarchyMessage(reason: string): string { + return SUPERCATEGORY_INVALID_WARNING.replace('{reason}', () => reason); +} + +function confidencePairsFromProb( + prob: unknown, + orderedNames: readonly (string | undefined)[], +): [string, number][] | null { + if (!Array.isArray(prob) || prob.length !== orderedNames.length) { + return null; + } + const pairs: [string, number][] = []; + prob.forEach((value, index) => { + const name = orderedNames[index]; + if (typeof name !== 'string' || !name || typeof value !== 'number' || !Number.isFinite(value)) { + return; + } + const clamped = Math.min(1, Math.max(0, value)); + if (clamped > PROB_EPSILON) { + pairs.push([name, clamped]); + } + }); + pairs.sort((left, right) => right[1] - left[1]); + return pairs.length ? pairs.slice(0, PROB_TOP_K) : null; +} + +function confidencePairsFromDiveExtension(value: unknown): [string, number][] | undefined { + if (!Array.isArray(value) || !value.length) { + return undefined; + } + const pairs: [string, number][] = []; + const names = new Set(); + const valid = value.every((pair) => { + if (!Array.isArray(pair) || pair.length !== 2 + || typeof pair[0] !== 'string' || !pair[0] + || typeof pair[1] !== 'number' || !Number.isFinite(pair[1]) + || pair[1] < 0 || pair[1] > 1 || names.has(pair[0])) { + return false; + } + names.add(pair[0]); + pairs.push([pair[0], pair[1]]); + return true; + }); + return valid ? pairs : undefined; +} + function hasValidBbox(annotation: CocoAnnotation): boolean { const { bbox } = annotation; return Array.isArray(bbox) && bbox.length === 4; @@ -109,6 +197,8 @@ type CocoAnnotation = { bbox?: [number, number, number, number]; score?: number; track_id?: number; + prob?: unknown; + dive_confidence_pairs?: unknown; /** * COCO `iscrowd` flag (0 or 1). In the COCO spec, 0 means a single instance with * polygon `segmentation` ([[x1, y1, ...]]); 1 means a crowd region whose @@ -220,8 +310,37 @@ function isCocoJson(value: unknown): value is CocoDocument { && Array.isArray(document.categories); } +function typeHierarchyFromCategories( + document: CocoDocument, +): { hierarchy?: Record; warnings: string[] } { + const warnings: string[] = []; + if (document.categories.some((category) => Array.isArray(category.parents) + && category.parents.length > 1)) { + warnings.push(SUPERCATEGORY_MULTI_PARENT_WARNING); + } + const names = document.categories.map((category) => category.name); + if (names.some((name) => typeof name !== 'string' || !name)) { + warnings.push(CATEGORY_MISSING_NAME_WARNING); + } + if (hasDuplicateCategoryNames(names)) { + warnings.push(SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING); + return { warnings }; + } + const hierarchy: Record = {}; + document.categories.forEach(({ name, supercategory }) => { + if (typeof name === 'string' && name + && typeof supercategory === 'string' && supercategory + && name !== supercategory) { + hierarchy[name] = supercategory; + } + }); + return Object.keys(hierarchy).length ? { hierarchy, warnings } : { warnings }; +} + function imageFrameMap(document: CocoDocument): Record { - const sorted = [...document.images].sort((a, b) => a.file_name.localeCompare(b.file_name, undefined, { numeric: true })); + const sorted = [...document.images].sort( + (a, b) => strNumericCompare(a.file_name, b.file_name), + ); const map: Record = {}; sorted.forEach((img, idx) => { map[img.id] = img.frame_index ?? idx; @@ -235,9 +354,15 @@ async function parseFile(path: string): Promise<[AnnotationSchema, Record [c.id, c])); + const orderedCategoryNames = parsed.categories.map((category) => category.name); + const duplicateCategoryNames = hasDuplicateCategoryNames(orderedCategoryNames); const frameByImageId = imageFrameMap(parsed); const tracks: AnnotationSchema['tracks'] = {}; + const classificationSourceByTrack = new Map(); let skippedRleMasks = false; + let probLengthMismatch = false; + let probIgnoredForDuplicates = false; + let diveConfidencePairsInvalid = false; validateAnnotationBounds(parsed.annotations); @@ -248,7 +373,32 @@ async function parseFile(path: string): Promise<[AnnotationSchema, Record classificationSource.frame + || (frame === classificationSource.frame && annotation.id > classificationSource.annotationId)) { + track.confidencePairs = confidencePairs; + classificationSourceByTrack.set(trackId, { frame, annotationId: annotation.id }); + } }); const annotations: AnnotationSchema = { version: 2, tracks, groups: {} }; const processed = processTrackAttributes(Object.values(annotations.tracks)); - const warnings = skippedRleMasks ? [RLE_SEGMENTATION_WARNING] : []; + const warnings: string[] = []; + if (skippedRleMasks) warnings.push(RLE_SEGMENTATION_WARNING); + if (probLengthMismatch) warnings.push(PROB_LENGTH_MISMATCH_WARNING); + if (probIgnoredForDuplicates) warnings.push(PROB_DUPLICATE_CATEGORY_WARNING); + if (diveConfidencePairsInvalid) warnings.push(DIVE_CONFIDENCE_PAIRS_INVALID_WARNING); const meta: Record = { attributes: processed.attributes }; // Restore the per-dataset station metadata namespaced under `info.dive_dataset_info`; the // caller merges it into the dataset's metadata. Omitted when absent/empty. @@ -325,12 +487,19 @@ async function serializeFile( excludeBelowThreshold: false, }, ) { - const categories = new Map(); const images = new Map(); const annotations: CocoAnnotation[] = []; let annotationId = 1; const thresholds = meta.confidenceFilters || {}; const defaultThreshold = thresholds.default ?? 0; + const pairsByTrack = new Map(); + const categoryNames: string[] = []; + const addCategoryName = (name: string) => { + if (!categoryNames.includes(name)) { + categoryNames.push(name); + } + }; + const hierarchy = meta.typeHierarchy || {}; Object.values(data.tracks).forEach((track) => { const filteredPairs = track.confidencePairs.filter(([name, score]) => { @@ -339,9 +508,22 @@ async function serializeFile( return keepType && keepThreshold; }); if (!filteredPairs.length) return; - const [className, score] = [...filteredPairs].sort((a, b) => b[1] - a[1])[0]; - const categoryId = categories.get(className) || (categories.size + 1); - categories.set(className, categoryId); + const pairs = filteredPairs.map(([name, score]) => [name, score] as [string, number]); + pairsByTrack.set(track.id, pairs); + pairs.forEach(([name]) => addCategoryName(name)); + }); + + Object.keys(hierarchy).sort().forEach(addCategoryName); + Array.from(new Set(Object.values(hierarchy))).sort().forEach(addCategoryName); + const categories = new Map(categoryNames.map((name, index) => [name, index + 1])); + + Object.values(data.tracks).forEach((track) => { + const pairs = pairsByTrack.get(track.id); + if (!pairs) return; + const [className, score] = [...pairs].sort((a, b) => b[1] - a[1])[0]; + const categoryId = categories.get(className) as number; + const probabilityByName = new Map(pairs); + const prob = categoryNames.map((name) => probabilityByName.get(name) || 0); track.features.forEach((feature) => { if (!feature.bounds) return; @@ -361,6 +543,8 @@ async function serializeFile( track_id: track.id, bbox: [x1, y1, Math.max(0, x2 - x1), Math.max(0, y2 - y1)], score, + prob, + dive_confidence_pairs: pairs.map(([name, confidence]) => [name, confidence]), ...(feature.attributes ? { dive_detection_attributes: feature.attributes } : {}), ...(track.attributes ? { dive_track_attributes: track.attributes } : {}), ...(feature.notes && feature.notes.length > 0 ? { dive_notes: feature.notes } : {}), @@ -373,6 +557,7 @@ async function serializeFile( id, name, keypoints: ['head', 'tail'], + ...(hierarchy[name] ? { supercategory: hierarchy[name] } : {}), })); // datasetInfo rides in the `info` block + dive_extensions; omitted entirely when empty. const datasetInfo = meta.datasetInfo && !isEmpty(meta.datasetInfo) ? meta.datasetInfo : undefined; @@ -382,6 +567,7 @@ async function serializeFile( 'dive_detection_attributes', 'dive_track_attributes', 'dive_notes', + 'dive_confidence_pairs', ...(datasetInfo ? ['dive_dataset_info'] : []), ], ...(datasetInfo ? { dive_dataset_info: datasetInfo } : {}), @@ -397,7 +583,15 @@ async function serializeFile( } export { + CATEGORY_MISSING_NAME_WARNING, + DIVE_CONFIDENCE_PAIRS_INVALID_WARNING, + PROB_DUPLICATE_CATEGORY_WARNING, + PROB_LENGTH_MISMATCH_WARNING, + SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING, + SUPERCATEGORY_MULTI_PARENT_WARNING, + invalidCocoHierarchyMessage, isCocoJson, parseFile, serializeFile, + typeHierarchyFromCategories, }; diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 27f8a7133..49c81c007 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -238,6 +238,7 @@ export interface ConversionArgs extends JobArgs { type: JobType.Conversion; meta: JsonConfig; mediaList: [string, string][]; + importWarnings?: string[]; } /** IPC payload when a CLI open must wait on media conversion. */ diff --git a/client/platform/desktop/frontend/components/ImportMultiCamBatchDialog.vue b/client/platform/desktop/frontend/components/ImportMultiCamBatchDialog.vue index cf58792eb..9ed0accf0 100644 --- a/client/platform/desktop/frontend/components/ImportMultiCamBatchDialog.vue +++ b/client/platform/desktop/frontend/components/ImportMultiCamBatchDialog.vue @@ -33,7 +33,11 @@ export default defineComponent({ const recentsMeta = await api.loadConfig(conversionArgs.meta.id); setRecents(recentsMeta); // Batch imports skip the confirmation dialog that normally shows these. - return importPayload.importWarnings; + const warnings = Array.from(new Set([ + ...(importPayload.importWarnings || []), + ...(conversionArgs.importWarnings || []), + ])); + return warnings.length ? warnings : undefined; } return { diff --git a/client/platform/desktop/frontend/components/ImportStereoBatchDialog.vue b/client/platform/desktop/frontend/components/ImportStereoBatchDialog.vue index 6b8da1dc0..2a94ad2cc 100644 --- a/client/platform/desktop/frontend/components/ImportStereoBatchDialog.vue +++ b/client/platform/desktop/frontend/components/ImportStereoBatchDialog.vue @@ -42,7 +42,11 @@ export default defineComponent({ const recentsMeta = await api.loadConfig(conversionArgs.meta.id); setRecents(recentsMeta); // Batch imports skip the confirmation dialog that normally shows these. - return importPayload.importWarnings; + const warnings = Array.from(new Set([ + ...(importPayload.importWarnings || []), + ...(conversionArgs.importWarnings || []), + ])); + return warnings.length ? warnings : undefined; } return { diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 91ff6c06a..602b99fe0 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -6,7 +6,9 @@ import { import type { DatasetType, MultiCamImportArgs } from 'dive-common/apispec'; import { itemsPerPageOptions } from 'dive-common/constants'; -import { JobType, DesktopMediaImportResponse, Job } from 'platform/desktop/constants'; +import { + JobType, DesktopMediaImportResponse, Job, ConversionArgs, +} from 'platform/desktop/constants'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; @@ -65,6 +67,19 @@ export default defineComponent({ error, loading: checkingMedia, request, reset: resetError, } = useRequest(); + async function presentImportWarnings(imports: ConversionArgs[]) { + const warnings = Array.from(new Set(imports.flatMap(({ importWarnings }) => ( + importWarnings || [] + )))); + if (warnings.length) { + await prompt({ + title: 'Import Warnings', + text: warnings, + positiveButton: 'Okay', + }); + } + } + async function open(dstype: DatasetType | 'bulk' | 'text', directory = false) { bulkImport.value = false; @@ -96,6 +111,8 @@ export default defineComponent({ const imports = await request(async () => Promise.all(argsArray.map((args) => api.finalizeImport(args)))); pendingImportPayload.value = null; + await presentImportWarnings(imports); + imports.forEach(async (conversionArgs) => { // Queue conversion job if (conversionArgs.mediaList.length > 0) { @@ -113,6 +130,7 @@ export default defineComponent({ importing.value = true; await request(async () => { const conversionArgs = await api.finalizeImport(args); + await presentImportWarnings([conversionArgs]); pendingImportPayload.value = null; // close dialog if (conversionArgs.mediaList.length === 0) { router.push({ diff --git a/client/platform/web-girder/api/dataset.service.ts b/client/platform/web-girder/api/dataset.service.ts index 81285ec46..6324d5250 100644 --- a/client/platform/web-girder/api/dataset.service.ts +++ b/client/platform/web-girder/api/dataset.service.ts @@ -338,12 +338,16 @@ export interface CreateMulticamDatasetArgs { metadataFileId?: string; } +export interface CreateMulticamDatasetResponse extends GirderModel { + importWarnings?: string[]; +} + function createMulticamDataset(args: CreateMulticamDatasetArgs) { const { parentFolderId, name, fps, type, subType, defaultDisplay, cameras, cameraOrder, calibrationFileId, metadataFileId, } = args; - return girderRest.post( + return girderRest.post( 'dive_dataset/multicam', { name, diff --git a/client/platform/web-girder/views/Upload.spec.ts b/client/platform/web-girder/views/Upload.spec.ts index 249c72d06..d6ad6000f 100644 --- a/client/platform/web-girder/views/Upload.spec.ts +++ b/client/platform/web-girder/views/Upload.spec.ts @@ -11,9 +11,16 @@ import { import type { ValidatedUploadRoleMap, ValidationResponse } from 'platform/web-girder/api'; import type { DatasetType } from 'dive-common/apispec'; import { openFromDisk } from 'platform/web-girder/utils'; -import { validateUploadGroup } from 'platform/web-girder/api'; +import { + createGirderFolder, + createMulticamDataset, + validateUploadGroup, + waitForFolderDatasetReady, +} from 'platform/web-girder/api'; import Upload from './Upload.vue'; +const prompt = vi.hoisted(() => vi.fn()); + Vue.config.ignoredElements = [/^v-/]; vi.mock('platform/web-girder/api', () => ({ @@ -38,7 +45,7 @@ vi.mock('vue-router/composables', () => ({ })); vi.mock('dive-common/vue-utilities/prompt-service', () => ({ - usePrompt: () => ({ prompt: vi.fn() }), + usePrompt: () => ({ prompt }), })); const Stub = { @@ -57,6 +64,16 @@ function uploadGirderStub(upload: () => Promise) { }; } +function multicamUploadGirderStub(uploadCameraDataset: () => Promise) { + return { + name: 'UploadGirder', + methods: { uploadCameraDataset }, + render(this: Vue, h: CreateElement) { + return h('div', this.$scopedSlots.default?.({ upload: () => Promise.resolve() })); + }, + }; +} + function file(name: string): File { return new File([name], name, { type: 'application/octet-stream' }); } @@ -88,7 +105,10 @@ function rejection(message: string): ValidationResponse { * @vue/test-utils' `mount()` typings do not resolve a setup-returned instance shape, so the * component is mounted through a host that captures the real, fully-typed instance via `ref`. */ -function mountUpload(upload: () => Promise = () => Promise.resolve()) { +function mountUpload( + upload: () => Promise = () => Promise.resolve(), + uploadGirder = uploadGirderStub(upload), +) { let child: InstanceType | undefined; const Host = defineComponent({ setup: () => () => h(Upload, { @@ -105,7 +125,7 @@ function mountUpload(upload: () => Promise = () => Promise.resolve()) { ImportButton: Stub, ImportMultiCamDialog: Stub, ImportMultiCamBatchDialog: Stub, - UploadGirder: uploadGirderStub(upload), + UploadGirder: uploadGirder, }, }); if (!child) { @@ -127,6 +147,7 @@ describe('Upload pending rows', () => { beforeEach(() => { vi.mocked(openFromDisk).mockReset(); vi.mocked(validateUploadGroup).mockReset(); + prompt.mockReset(); }); it('derives fan-out from the server media role, not the client slot guess', async () => { @@ -337,10 +358,10 @@ describe('Upload pending rows', () => { }), } as never); - const wrapper = mountUpload(); - await wrapper.vm.openImport('video'); + const { vm } = mountUpload(); + await vm.openImport('video'); - const [row] = wrapper.vm.pendingUploads; + const [row] = vm.pendingUploads; expect(row.uploadFiles.map((entry: File) => entry.name)).toEqual([ 'dive.mp4', 'hierarchy.config.json', @@ -362,4 +383,41 @@ describe('Upload pending rows', () => { expect(upload).toHaveBeenCalledTimes(1); }); + + it('surfaces import warnings returned when the multicam dataset is linked', async () => { + const uploadCameraDataset = vi.fn().mockResolvedValue({ + folder: { _id: 'camera-folder' }, + jobIds: [], + }); + vi.mocked(createGirderFolder).mockResolvedValue({ data: { _id: 'dataset-folder' } } as never); + vi.mocked(validateUploadGroup).mockResolvedValue({ + data: validation({ roles: { media: ['left.mp4'] } }), + } as never); + vi.mocked(waitForFolderDatasetReady).mockResolvedValue(undefined as never); + vi.mocked(createMulticamDataset).mockResolvedValue({ + data: { _id: 'parent-folder', importWarnings: ['Camera hierarchy was skipped.'] }, + } as never); + const { vm } = mountUpload( + () => Promise.resolve(), + multicamUploadGirderStub(uploadCameraDataset), + ); + vm.registerSubfolderCameras([{ + cameraName: 'left', sourcePath: 'left-source', files: [file('left.mp4')], + }]); + + await vm.multiCamImport({ + datasetName: 'multicam', + defaultDisplay: 'left', + type: 'video', + sourceList: { + left: { sourcePath: 'left-source', trackFile: '', type: 'video' }, + }, + }); + + expect(prompt).toHaveBeenCalledWith({ + title: 'Import Warnings', + text: ['Camera hierarchy was skipped.'], + positiveButton: 'OK', + }); + }); }); diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue index 8fb9d7b59..34c0e205d 100644 --- a/client/platform/web-girder/views/Upload.vue +++ b/client/platform/web-girder/views/Upload.vue @@ -649,6 +649,14 @@ export default defineComponent({ }); multicamLinked = true; + if (parentFolder.importWarnings?.length) { + await prompt({ + title: 'Import Warnings', + text: parentFolder.importWarnings, + positiveButton: 'OK', + }); + } + if (registrationSeed?.values) { // Seed the dataset's saved camera registration (the same // registration the in-app panel edits and the Align button diff --git a/docs/DataFormats.md b/docs/DataFormats.md index cd06354a0..44789cfe1 100644 --- a/docs/DataFormats.md +++ b/docs/DataFormats.md @@ -210,7 +210,8 @@ DIVE Configuration JSON exports include a valid non-empty hierarchy and omit an one. The `config.json` embedded in a dataset zip follows the same rules. Invalid stored hierarchy prevents either configuration export and reports `Type hierarchy is invalid: {reason}. No configuration file was exported.` Hierarchy is not -transported by DIVE Annotation JSON, COCO/KWCOCO, VIAME CSV, KPF, NIST, or `labels.txt`. +transported by DIVE Annotation JSON, VIAME CSV, KPF, NIST, or `labels.txt`. KWCOCO transports it +through category `supercategory` fields as described below. When importing a DIVE Configuration JSON with `datasetInfo`, **Overwrite** import (the default) replaces the existing `datasetInfo` block; an additive import merges it per-key @@ -349,6 +350,35 @@ are also accepted on import. * For video datasets, DIVE exports per-frame synthetic names (for example, `frame_000123.jpg`) because base COCO does not define a canonical video container field. +### DIVE KWCOCO Classification Profile + +DIVE Web and Desktop use the same KWCOCO profile for hierarchy and complete confidence vectors: + +* `categories` contains every type in an exported confidence vector and every child or parent in + the dataset hierarchy. A child's immediate parent is written as `supercategory`. +* Every annotation retains standard `category_id` and `score` fields for the highest-confidence + exported pair, so readers that ignore KWCOCO extensions still receive a primary category. +* Every annotation also has a dense `prob` array aligned by position with the document's complete + `categories` array. +* `dive_confidence_pairs` stores the track's ordered sparse vector exactly. This preserves the + difference between a missing pair and a pair explicitly scored `0`, which a dense `prob` array + cannot express. The extension is listed in `info.dive_extensions` and takes precedence when a + DIVE-authored file is imported again. A present but malformed extension produces one import + warning and falls back to a valid `prob` vector or the primary category and score. + +For an external KWCOCO file without `dive_confidence_pairs`, DIVE maps `prob` by the original +category-array order, including unnamed positional slots. It accepts finite numeric values, clamps +them to `[0, 1]`, keeps the ten highest entries above `0.001`, and falls back to `category_id` plus +`score` when the vector length is wrong or duplicate category names make the mapping ambiguous. +For a track whose annotations contain different vectors, the annotation at the highest frame index +wins; the greater annotation ID wins a same-frame tie, independent of file order. + +Categories with missing names, duplicate names, multiple parents, invalid edges, or cycles produce +an import warning. Usable annotations are still imported. In a multicamera import, the first valid +camera hierarchy in configured camera order becomes the parent dataset hierarchy. Matching later +hierarchies coalesce; conflicting later hierarchies are skipped with a warning. Camera datasets do +not retain separate hierarchy copies. + ### DIVE COCO Attribute Extensions COCO does not define standard fields for arbitrary track or detection attributes @@ -361,7 +391,7 @@ export/import, DIVE uses extension fields on each COCO `annotation` object: These extension keys are declared in the COCO `info` object as: -* `info.dive_extensions = ["dive_detection_attributes", "dive_track_attributes", "dive_notes"]` +* `info.dive_extensions = ["dive_detection_attributes", "dive_track_attributes", "dive_notes", "dive_confidence_pairs"]` ### Dataset-level metadata (`datasetInfo`) @@ -402,7 +432,9 @@ For COCO files produced by DIVE: * DIVE writes `dive_detection_attributes` and `dive_track_attributes` on each annotation when attributes are present. * DIVE writes `dive_notes` on each annotation when that feature has a note. -* Re-importing that file into DIVE preserves those attributes and notes. +* DIVE writes category-aligned `prob` plus exact `dive_confidence_pairs` on each annotation. +* Re-importing that file into DIVE preserves hierarchy edges, track IDs, complete confidence + vectors, attributes, and notes. For COCO files not produced by DIVE: @@ -428,22 +460,25 @@ For COCO files not produced by DIVE: { "info": { "description": "DIVE export for my-dataset", - "dive_extensions": ["dive_detection_attributes", "dive_track_attributes", "dive_notes"] + "dive_extensions": ["dive_detection_attributes", "dive_track_attributes", "dive_notes", "dive_confidence_pairs"] }, "images": [ { "id": 1, "file_name": "frame_000000.jpg", "frame_index": 0 } ], "categories": [ { "id": 1, "name": "fish", "keypoints": ["head", "tail"] }, - { "id": 2, "name": "crab" } + { "id": 2, "name": "shark", "supercategory": "fish" }, + { "id": 3, "name": "crab" } ], "annotations": [ { "id": 1, "image_id": 1, - "category_id": 1, + "category_id": 2, "bbox": [100, 200, 50, 80], "score": 0.97, + "prob": [0.03, 0.97, 0], + "dive_confidence_pairs": [["shark", 0.97], ["fish", 0.03]], "track_id": 42, "dive_detection_attributes": { "visibility": "poor", @@ -458,9 +493,11 @@ For COCO files not produced by DIVE: { "id": 2, "image_id": 1, - "category_id": 2, + "category_id": 3, "bbox": [320, 140, 120, 90], "score": 0.91, + "prob": [0, 0, 0.91], + "dive_confidence_pairs": [["crab", 0.91]], "track_id": 77, "segmentation": [ [320, 140, 360, 130, 430, 170, 440, 220, 360, 230, 325, 200] diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index aa9d1ff5c..d86c7457d 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -1,7 +1,7 @@ import copy import json from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Literal, Optional, Set, Tuple +from typing import Any, Dict, Generator, Iterable, List, Literal, Optional, Set, Tuple, Union from bson.objectid import InvalidId, ObjectId import cherrypy @@ -661,6 +661,28 @@ def remove_camera_type_hierarchy(folder: types.GirderModel) -> bool: return True +def promote_camera_type_hierarchies( + parent_folder: types.GirderModel, + loaded_children: Dict[str, types.GirderModel], + camera_order: List[str], +) -> Tuple[Optional[Dict[str, str]], List[str]]: + """Resolve camera hierarchies into the parent before removing child copies.""" + promoted = fromMeta(parent_folder, 'typeHierarchy') + warnings = [] + for name in camera_order: + child_hierarchy = fromMeta(loaded_children[name], 'typeHierarchy') + if child_hierarchy is None: + continue + try: + write = resolve_type_hierarchy(promoted, True, child_hierarchy, 'additive') + except TypeHierarchyError as error: + warnings.append(f'Camera "{name}" type hierarchy was skipped: {error.reason}') + continue + if write['action'] == 'set': + promoted = write['hierarchy'] + return promoted, warnings + + def type_hierarchy_for_export( dsFolder: types.GirderModel, user: Optional[types.GirderUserModel] = None, @@ -742,16 +764,29 @@ def _filtered_annotation_tracks( annotations = crud_annotation.get_annotations(dsFolder, revision=revision) tracks = annotations['tracks'] thresholds = fromMeta(dsFolder, "confidenceFilters", {}) if excludeBelowThreshold else {} + default_threshold = thresholds.get('default', 0) updated_tracks = {} for track_id in tracks: track = models.Track(**tracks[track_id]) - if excludeBelowThreshold and not track.exceeds_thresholds(thresholds): - continue + confidence_pairs = track.confidencePairs + if excludeBelowThreshold: + confidence_pairs = [ + pair + for pair in confidence_pairs + if pair[1] >= thresholds.get(pair[0], default_threshold) + ] if typeFilter: - confidence_pairs = [item for item in track.confidencePairs if item[0] in typeFilter] - if not confidence_pairs: - continue - updated_tracks[track_id] = tracks[track_id] + confidence_pairs = [pair for pair in confidence_pairs if pair[0] in typeFilter] + if not confidence_pairs: + continue + if excludeBelowThreshold or typeFilter: + # Filters select raw stored evidence. Copy before pruning so an export + # never mutates the stored track or leaks removed pairs into its output. + exported_track = dict(tracks[track_id]) + exported_track['confidencePairs'] = [list(pair) for pair in confidence_pairs] + updated_tracks[track_id] = exported_track + else: + updated_tracks[track_id] = tracks[track_id] return updated_tracks @@ -794,6 +829,7 @@ def _coco_json_export_text( image_filenames=image_filenames, dataset_name=dsFolder['name'], datasetInfo=fromMeta(dsFolder, "datasetInfo", {}), + typeHierarchy=type_hierarchy_for_export(dsFolder, user), ) return json.dumps(coco) @@ -810,10 +846,23 @@ def export_multicam_annotations_zipstream( if format not in ('viame_csv', 'dive_json', 'coco_json'): raise RestException(f'Format {format} is not a valid option.') + if format == 'coco_json': + type_hierarchy_for_export(dsFolder, user) + multi_cam = fromMeta(dsFolder, constants.MultiCamMarker) or {} + children = {} + for cam_name in _multicam_camera_order(multi_cam): + cam_info = multi_cam['cameras'][cam_name] + child = Folder().load(cam_info['folderId'], level=AccessType.READ, user=user) + if child is None: + raise RestException( + f'Camera folder for "{cam_name}" was not found', + code=404, + ) + children[cam_name] = child + def stream(): z = ziputil.ZipGenerator() zip_path = f"./{dsFolder['name']}/" - multi_cam = fromMeta(dsFolder, constants.MultiCamMarker) or {} def makeMultiCamJson(): yield json.dumps(multi_cam, indent=2).encode('utf-8') @@ -823,13 +872,7 @@ def makeMultiCamJson(): nested_type_filter = typeFilter if typeFilter is not None else set() for cam_name in _multicam_camera_order(multi_cam): - cam_info = multi_cam['cameras'][cam_name] - child = Folder().load(cam_info['folderId'], level=AccessType.READ, user=user) - if child is None: - raise RestException( - f'Camera folder for "{cam_name}" was not found', - code=404, - ) + child = children[cam_name] child_path = f'{zip_path}{cam_name}/' if format == 'viame_csv': _, gen = crud_annotation.get_annotation_csv_generator( @@ -979,25 +1022,9 @@ def makeMetajson(): def makeDiveJson(): """Include DIVE JSON output annotation file""" annotations = crud_annotation.get_annotations(dsFolder) - tracks = annotations['tracks'] - thresholds = None - if excludeBelowThreshold: - thresholds = fromMeta(dsFolder, "confidenceFilters", {}) - if thresholds is None: - thresholds = {} - - updated_tracks = {} - for t in tracks: - track = models.Track(**tracks[t]) - if (not excludeBelowThreshold) or track.exceeds_thresholds(thresholds): - if typeFilter: - confidence_pairs = [ - item for item in track.confidencePairs if item[0] in typeFilter - ] - if not confidence_pairs: - continue - updated_tracks[t] = tracks[t] - annotations['tracks'] = updated_tracks + annotations['tracks'] = _filtered_annotation_tracks( + dsFolder, None, excludeBelowThreshold, typeFilter + ) yield json.dumps(annotations) for data in z.addFile(makeMetajson, Path(f'{zip_path}{constants.ConfigFileName}')): @@ -1547,7 +1574,7 @@ def create_multicam( user: types.GirderUserModel, parent_folder: types.GirderModel, data: dict, -) -> types.GirderModel: +) -> Union[types.GirderModel, Dict[str, Any]]: """Finalize a multicam dataset whose camera folders already live under parent_folder.""" validated: CreateMulticamArgs = crud.get_validated_model(CreateMulticamArgs, **data) if parent_folder['name'] != validated.name: @@ -1658,12 +1685,14 @@ def create_multicam( # Frame alignment pairs frames across cameras downstream, so a per-camera # frame-count equality check would reject the primary use case for this feature. + promoted_hierarchy, hierarchy_warnings = promote_camera_type_hierarchies( + parent_folder, loaded_children, camera_order + ) default_child = loaded_children[validated.defaultDisplay] parent_folder_doc = parent_folder multi_cam_cameras: Dict[str, Dict[str, str]] = {} for name in camera_order: child = loaded_children[name] - remove_camera_type_hierarchy(child) if child['name'] != name: child['name'] = name Folder().save(child) @@ -1715,6 +1744,7 @@ def create_multicam( } parent_folder_doc['meta'] = { **mutable_meta, + **({'typeHierarchy': promoted_hierarchy} if promoted_hierarchy else {}), constants.DatasetMarker: True, constants.TypeMarker: constants.MultiType, constants.SubTypeMarker: validated.subType, @@ -1755,7 +1785,15 @@ def create_multicam( {'default': 0.1}, ) Folder().save(parent_folder_doc) + # The parent is now the durable canonical owner. Do not remove the only hierarchy + # copies from camera folders until every fallible validation above has succeeded. + for child in loaded_children.values(): + remove_camera_type_hierarchy(child) crud.get_or_create_auxiliary_folder(parent_folder_doc, user) + if hierarchy_warnings: + response: Dict[str, Any] = dict(parent_folder_doc) + response['importWarnings'] = hierarchy_warnings + return response return parent_folder_doc diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 65263406c..70f18dfe2 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta import json -from typing import Dict, List, Literal, Optional, Tuple, TypedDict, cast +from typing import Dict, List, Literal, NamedTuple, Optional, Tuple, TypedDict, cast from girder.constants import AccessType from girder.exceptions import RestException @@ -14,6 +14,7 @@ from girder_plugin_worker.status import CustomJobStatus from pydantic import BaseModel import pymongo +from typing_extensions import NotRequired from dive_server import crud, crud_annotation, crud_dataset from dive_tasks import tasks @@ -594,6 +595,7 @@ def run_training( 'meta': Optional[dict], 'attributes': Optional[dict], 'type': crud.FileType, + 'hierarchy': NotRequired[Optional[Dict[str, str]]], }, ) @@ -644,13 +646,14 @@ def _get_data_by_type( as_type = crud.FileType.COCO_JSON elif models.MetadataMutable.is_dive_configuration(data_dict): hierarchy_present = 'typeHierarchy' in data_dict - normalized_hierarchy = None + raw_hierarchy = data_dict.get('typeHierarchy') if hierarchy_present: - normalized_hierarchy = normalize_type_hierarchy(data_dict['typeHierarchy']) + normalize_type_hierarchy(raw_hierarchy) data_dict = models.MetadataMutable(**data_dict).dict(exclude_none=True) if hierarchy_present: - # Pydantic drops explicit null; preserve presence for the resolver. - data_dict['typeHierarchy'] = normalized_hierarchy + # Pydantic drops explicit null. Preserve the raw instruction so additive + # import can distinguish an empty no-op from an explicit deletion. + data_dict['typeHierarchy'] = raw_hierarchy as_type = crud.FileType.DIVE_CONF else: as_type = crud.FileType.DIVE_JSON @@ -659,7 +662,7 @@ def _get_data_by_type( else: raise RestException('Got file of unknown and unusable type') - if configuration_only and as_type != crud.FileType.DIVE_CONF: + if configuration_only and as_type not in (crud.FileType.DIVE_CONF, crud.FileType.COCO_JSON): return None, None # Parse the file as the now known type @@ -700,12 +703,14 @@ def _get_data_by_type( coco_warnings, datasetInfo, ) = kwcoco.load_coco_as_tracks_and_attributes(data_dict) + hierarchy, hierarchy_warnings = kwcoco.type_hierarchy_from_categories(data_dict) return { 'annotations': converted, 'meta': {"datasetInfo": datasetInfo} if datasetInfo else None, 'attributes': attributes, 'type': as_type, - }, coco_warnings or warnings + 'hierarchy': hierarchy, + }, (coco_warnings + hierarchy_warnings) or warnings if as_type == crud.FileType.DIVE_CONF: return { 'annotations': None, @@ -835,27 +840,40 @@ def _fresh_folder_snapshot(folder: types.GirderModel) -> types.GirderModel: return cast(types.GirderModel, fresh) if isinstance(fresh, dict) else folder +class HierarchyInstruction(NamedTuple): + present: bool + hierarchy: object + soft_warning: Optional[str] = None + + def _resolve_configuration_hierarchy( existing: object, - instructions: list, + instructions: List[HierarchyInstruction], additive: bool, -) -> HierarchyWrite: +) -> Tuple[HierarchyWrite, List[str]]: candidate = existing final_write: HierarchyWrite = {'action': 'none'} - for incoming_present, incoming in instructions: - next_write = resolve_type_hierarchy( - candidate, - incoming_present, - incoming, - 'additive' if additive else 'overwrite', - ) + soft_warnings: List[str] = [] + for instruction in instructions: + try: + next_write = resolve_type_hierarchy( + candidate, + instruction.present, + instruction.hierarchy, + 'additive' if additive or instruction.soft_warning is not None else 'overwrite', + ) + except TypeHierarchyError as error: + if instruction.soft_warning is None: + raise + soft_warnings.append(instruction.soft_warning.format(reason=error.reason)) + continue if next_write['action'] == 'set': candidate = next_write['hierarchy'] final_write = next_write elif next_write['action'] == 'delete': candidate = None final_write = next_write - return final_write + return final_write, soft_warnings def _prepare_configuration_imports( @@ -890,10 +908,23 @@ def _prepare_configuration_imports( if results is None: continue parsed_json_items[str(item['_id'])] = (file, results, warnings) + if results['type'] == crud.FileType.COCO_JSON: + coco_hierarchy = results.get('hierarchy') + if coco_hierarchy is not None: + hierarchy_instructions.append( + HierarchyInstruction( + True, + coco_hierarchy, + kwcoco.SUPERCATEGORY_INVALID_WARNING, + ) + ) + continue if results['type'] != crud.FileType.DIVE_CONF: continue meta = results['meta'] or {} - hierarchy_instructions.append(('typeHierarchy' in meta, meta.get('typeHierarchy'))) + hierarchy_instructions.append( + HierarchyInstruction('typeHierarchy' in meta, meta.get('typeHierarchy')) + ) config_results.append(results) if ( @@ -907,7 +938,7 @@ def _prepare_configuration_imports( ) try: - hierarchy_write = _resolve_configuration_hierarchy( + hierarchy_write, _ = _resolve_configuration_hierarchy( fromMeta(canonical, 'typeHierarchy'), hierarchy_instructions, additive, @@ -990,13 +1021,15 @@ def _apply_configuration_imports( if promoted_write['action'] == 'set': existing_hierarchy = promoted_write['hierarchy'] try: - hierarchy_write = _resolve_configuration_hierarchy( + hierarchy_write, soft_warnings = _resolve_configuration_hierarchy( existing_hierarchy, configuration_plan['hierarchy_instructions'], configuration_plan['additive'], ) except TypeHierarchyError as error: raise crud.hierarchy_rest_error(error) from error + if soft_warnings: + configuration_plan.setdefault('warnings', []).extend(soft_warnings) if hierarchy_write['action'] == 'none': hierarchy_write = promoted_write diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index b3eae43e7..db5039e00 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -368,6 +368,10 @@ class CocoMetadata(BaseModel): images: Dict[int, dict] videos: Dict[int, dict] datasetInfo: types.DatasetInfo = {} + # KWCOCO ``prob`` arrays align positionally with the document's categories array, + # rather than the id-keyed category lookup above. Keep unnamed slots so that + # vector length validation remains meaningful. + ordered_category_names: List[Optional[str]] = Field(default_factory=list) class BrandData(BaseModel): diff --git a/server/dive_utils/serializers/kwcoco.py b/server/dive_utils/serializers/kwcoco.py index 032bcf2bf..f7d55d89b 100644 --- a/server/dive_utils/serializers/kwcoco.py +++ b/server/dive_utils/serializers/kwcoco.py @@ -6,6 +6,7 @@ """ import functools +import math from typing import Any, Dict, Iterable, List, Optional, Tuple from dive_utils import constants, strNumericCompare, types @@ -18,6 +19,120 @@ 'Bounding boxes and other annotation data were imported, but masks were skipped.' ) +PROB_TOP_K = 10 +PROB_EPSILON = 0.001 + +PROB_LENGTH_MISMATCH_WARNING = ( + 'Some annotations had a "prob" array whose length did not match the number of categories. ' + 'Class probabilities were ignored for those annotations; the primary category and score ' + 'were imported instead.' +) +PROB_DUPLICATE_CATEGORY_WARNING = ( + 'The COCO file contains duplicate category names, so "prob" arrays cannot be mapped to ' + 'class names. Class probabilities were ignored; primary categories and scores were ' + 'imported instead.' +) +DIVE_CONFIDENCE_PAIRS_WARNING = ( + 'Some annotations had malformed "dive_confidence_pairs" values. Those values were ' + 'ignored; the primary category and score or a valid "prob" vector were imported instead.' +) +SUPERCATEGORY_MULTI_PARENT_WARNING = ( + 'Some COCO categories declare multiple parents via "parents", which DIVE cannot ' + 'represent. Only single-parent "supercategory" edges were imported.' +) +SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING = ( + 'The COCO file contains duplicate category names, so category hierarchy edges cannot be ' + 'mapped to class names. The dataset type hierarchy was left unchanged.' +) +CATEGORY_MISSING_NAME_WARNING = ( + 'Some COCO categories have no non-empty string name. Those positional category slots were ' + 'ignored when importing classifications and hierarchy edges.' +) +SUPERCATEGORY_INVALID_WARNING = ( + 'The category hierarchy in the COCO file could not be applied: {reason}. ' + 'Annotations were imported without changing the dataset type hierarchy.' +) + + +def _has_duplicate_names(names: List[Optional[str]]) -> bool: + usable_names = [name for name in names if isinstance(name, str) and name] + return len(set(usable_names)) != len(usable_names) + + +def _confidence_pairs_from_prob( + prob: List[Any], ordered_names: List[Optional[str]] +) -> Optional[List[Tuple[str, float]]]: + """Map a positional KWCOCO probability vector to DIVE confidence pairs.""" + if len(prob) != len(ordered_names): + return None + pairs = [ + (name, min(1.0, max(0.0, float(value)))) + for name, value in zip(ordered_names, prob) + if isinstance(name, str) + and name + and isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ] + pairs = [pair for pair in pairs if pair[1] > PROB_EPSILON] + pairs.sort(key=lambda pair: pair[1], reverse=True) + return pairs[:PROB_TOP_K] or None + + +def _confidence_pairs_from_extension(value: Any) -> Optional[List[Tuple[str, float]]]: + """Read DIVE's sparse confidence extension without pruning zero-valued pairs.""" + if not isinstance(value, list) or not value: + return None + pairs: List[Tuple[str, float]] = [] + names = set() + for pair in value: + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + return None + name, confidence = pair + if ( + not isinstance(name, str) + or not name + or name in names + or not isinstance(confidence, (int, float)) + or isinstance(confidence, bool) + or not math.isfinite(confidence) + or confidence < 0 + or confidence > 1 + ): + return None + names.add(name) + pairs.append((name, float(confidence))) + return pairs + + +def type_hierarchy_from_categories( + coco: Dict[str, Any], +) -> Tuple[Optional[Dict[str, str]], List[str]]: + """Derive DIVE child-to-parent edges from KWCOCO category supercategories.""" + categories = coco.get('categories', []) + warnings: List[str] = [] + if any( + isinstance(category.get('parents'), list) and len(category['parents']) > 1 + for category in categories + ): + warnings.append(SUPERCATEGORY_MULTI_PARENT_WARNING) + + names = [category.get('name') for category in categories] + if any(not isinstance(name, str) or not name for name in names): + warnings.append(CATEGORY_MISSING_NAME_WARNING) + if _has_duplicate_names(names): + warnings.append(SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING) + return None, warnings + + hierarchy: Dict[str, str] = {} + for category in categories: + name = category.get('name') + parent = category.get('supercategory') + # Some producers spell roots as a self-supercategory; it is not an edge. + if isinstance(name, str) and name and isinstance(parent, str) and parent and name != parent: + hierarchy[name] = parent + return hierarchy or None, warnings + def _has_valid_bbox(annotation: dict) -> bool: bbox = annotation.get('bbox') @@ -152,7 +267,8 @@ def _parse_annotation( category_id = annotation['category_id'] score = annotation.get('score', 1.0) # may not exist, default to 1.0 - class_name = meta.categories[category_id]['name'] + category = meta.categories.get(category_id, {}) + class_name = category.get('name') or 'unknown' confidence_pair = (class_name, score) # parse keypoints @@ -160,7 +276,7 @@ def _parse_annotation( head_tail = [] for keypoint in keypoints: if isinstance(keypoint, (int, float)): # [x1, y1, v1, ...] coco format - keypoint_labels = meta.categories[category_id].get('keypoints', []) + keypoint_labels = category.get('keypoints', []) n = min(len(keypoint_labels), int(len(keypoints) / 3)) # stopping index for i in range(n): point = keypoints[3 * i : 3 * i + 2] # extract [x, y] pair @@ -283,6 +399,7 @@ def file_name_cmp(item1, item2): images=images_map, videos=videos_map, datasetInfo=datasetInfo, + ordered_category_names=[category.get('name') for category in categories], ) @@ -303,6 +420,23 @@ def load_coco_as_tracks_and_attributes( annotations = coco.get('annotations', []) _validate_annotation_bounds(annotations) + ordered_names = meta.ordered_category_names + duplicate_category_names = _has_duplicate_names(ordered_names) + prob_length_mismatch = False + prob_ignored_for_duplicates = False + + # Process each logical track in frame order so confidence pairs describe its + # temporal endpoint regardless of annotation order in the source file. COCO + # annotation IDs make equal-frame selection deterministic as well. + annotations = sorted( + annotations, + key=lambda annotation: ( + meta.images[annotation['image_id']]['frame_index'], + annotation['id'], + ), + ) + + malformed_extension = False for annotation in annotations: ( feature, @@ -313,6 +447,23 @@ def load_coco_as_tracks_and_attributes( ) = _parse_annotation_for_tracks(annotation, meta) skipped_rle_masks = skipped_rle_masks or rle_skipped + extension_present = 'dive_confidence_pairs' in annotation + extension_pairs = _confidence_pairs_from_extension(annotation.get('dive_confidence_pairs')) + if extension_pairs is not None: + confidence_pairs = extension_pairs + else: + malformed_extension = malformed_extension or extension_present + prob = annotation.get('prob') + if isinstance(prob, list): + if duplicate_category_names: + prob_ignored_for_duplicates = True + else: + prob_pairs = _confidence_pairs_from_prob(prob, ordered_names) + if prob_pairs is None and len(prob) != len(ordered_names): + prob_length_mismatch = True + elif prob_pairs: + confidence_pairs = prob_pairs + trackId, _, frame, _ = annotation_info(annotation, meta) if trackId not in tracks: @@ -342,6 +493,12 @@ def load_coco_as_tracks_and_attributes( } if skipped_rle_masks: warnings.append(RLE_SEGMENTATION_WARNING) + if prob_length_mismatch: + warnings.append(PROB_LENGTH_MISMATCH_WARNING) + if prob_ignored_for_duplicates: + warnings.append(PROB_DUPLICATE_CATEGORY_WARNING) + if malformed_extension: + warnings.append(DIVE_CONFIDENCE_PAIRS_WARNING) return converted, metadata_attributes, warnings, meta.datasetInfo @@ -396,6 +553,7 @@ def export_dive_as_coco( image_filenames: Dict[int, str], dataset_name: str, datasetInfo: Optional[types.DatasetInfo] = None, + typeHierarchy: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: """ Export DIVE tracks to a single-dataset COCO JSON document. @@ -407,14 +565,30 @@ def export_dive_as_coco( datasetInfo: per-dataset station metadata; when present, written under ``info.dive_dataset_info`` and advertised in ``info.dive_extensions``. Omitted entirely when empty. + typeHierarchy: DIVE child-to-parent category hierarchy, emitted through + KWCOCO's ``categories[].supercategory`` field. """ - categories: Dict[str, int] = {} + parsed_tracks = [Track(**track_doc) for track_doc in tracks] + category_names: List[str] = [] + + def add_category_name(name: str) -> None: + if name not in category_names: + category_names.append(name) + + for track in parsed_tracks: + for name, _confidence in track.confidencePairs: + add_category_name(name) + for name in sorted((typeHierarchy or {}).keys()): + add_category_name(name) + for name in sorted(set((typeHierarchy or {}).values())): + add_category_name(name) + + categories = {name: index + 1 for index, name in enumerate(category_names)} coco_annotations: List[dict] = [] images: Dict[int, dict] = {} annotation_id = 1 - for track_doc in tracks: - track = Track(**track_doc) + for track in parsed_tracks: for feature in track.features: if feature.frame not in image_filenames: continue @@ -423,7 +597,7 @@ def export_dive_as_coco( if not track.confidencePairs: continue class_name, score = max(track.confidencePairs, key=lambda x: x[1]) - category_id = categories.setdefault(class_name, len(categories) + 1) + category_id = categories[class_name] x1, y1, x2, y2 = feature.bounds width = max(0, x2 - x1) height = max(0, y2 - y1) @@ -447,6 +621,11 @@ def export_dive_as_coco( # Single-instance polygon export; DIVE does not emit crowd RLE (iscrowd: 1). 'iscrowd': 0, 'score': score, + # KWCOCO probability vectors align with document category order. + 'prob': [dict(track.confidencePairs).get(name, 0.0) for name in category_names], + # Preserve sparse membership and explicit zero confidence without + # requiring consumers to infer it from a dense probability vector. + 'dive_confidence_pairs': [list(pair) for pair in track.confidencePairs], } # Keep a stable object identity across frames when track data exists. annotation['track_id'] = track.id @@ -467,6 +646,9 @@ def export_dive_as_coco( categories_doc: List[dict] = [] for class_name, category_id in categories.items(): category: Dict[str, Any] = {'id': category_id, 'name': class_name} + parent = (typeHierarchy or {}).get(class_name) + if parent is not None: + category['supercategory'] = parent # When keypoints are exported, publish the category labels explicitly. category['keypoints'] = ['head', 'tail'] categories_doc.append(category) @@ -477,6 +659,7 @@ def export_dive_as_coco( 'dive_detection_attributes', 'dive_track_attributes', 'dive_notes', + 'dive_confidence_pairs', ], } if datasetInfo: diff --git a/server/tests/test_coco_export_filter.py b/server/tests/test_coco_export_filter.py new file mode 100644 index 000000000..692a22b13 --- /dev/null +++ b/server/tests/test_coco_export_filter.py @@ -0,0 +1,31 @@ +from copy import deepcopy + +from dive_server import crud_annotation, crud_dataset + + +def test_type_filter_prunes_exported_confidence_pairs_without_mutating_storage(monkeypatch): + tracks = { + '7': { + 'id': 7, + 'begin': 0, + 'end': 0, + 'confidencePairs': [['fish', 0.8], ['shark', 0.4]], + 'attributes': {}, + 'features': [{'frame': 0, 'bounds': [1, 2, 3, 4]}], + } + } + monkeypatch.setattr( + crud_annotation, + 'get_annotations', + lambda _folder, revision=None: {'tracks': tracks}, + ) + + exported = crud_dataset._filtered_annotation_tracks({'meta': {}}, None, False, {'fish'}) + + assert exported == { + '7': { + **deepcopy(tracks['7']), + 'confidencePairs': [['fish', 0.8]], + } + } + assert tracks['7']['confidencePairs'] == [['fish', 0.8], ['shark', 0.4]] diff --git a/server/tests/test_create_multicam.py b/server/tests/test_create_multicam.py index f72394fac..71ed140b2 100644 --- a/server/tests/test_create_multicam.py +++ b/server/tests/test_create_multicam.py @@ -24,6 +24,29 @@ def _dataset_parent(): return {'_id': 'multi-id', 'name': 'stereo-set'} +def _saved_parent_meta(folder_cls): + return next( + call.args[0]['meta'] + for call in folder_cls.return_value.save.call_args_list + if call.args[0].get('_id') == 'multi-id' + ) + + +def _stereo_data(): + return { + 'name': 'stereo-set', + 'fps': 5, + 'type': 'image-sequence', + 'subType': 'stereo', + 'defaultDisplay': 'left', + 'cameraOrder': ['left', 'right'], + 'cameras': { + 'left': {'folderId': 'left-id'}, + 'right': {'folderId': 'right-id'}, + }, + } + + @patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') @patch('dive_server.crud_dataset.Folder') @patch('dive_server.crud_dataset.crud.valid_images') @@ -67,7 +90,7 @@ def load_folder(folder_id, level=None, user=None): assert result == dataset_parent folder_cls.return_value.createFolder.assert_not_called() folder_cls.return_value.move.assert_not_called() - saved_meta = folder_cls.return_value.save.call_args_list[-1][0][0]['meta'] + saved_meta = _saved_parent_meta(folder_cls) assert saved_meta[constants.TypeMarker] == constants.MultiType assert saved_meta[constants.SubTypeMarker] == 'stereo' assert saved_meta[constants.MultiCamMarker]['cameraOrder'] == ['left', 'right'] @@ -77,6 +100,94 @@ def load_folder(folder_id, level=None, user=None): assert saved_meta['confidenceFilters'] == {'default': 0.7, 'salmon': 0.85} +@patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.valid_images') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_create_multicam_promotes_camera_hierarchies_to_the_parent( + _verify, valid_images_mock, folder_cls, _aux +): + parent = _dataset_parent() + parent['meta'] = {} + left = _child_folder('left-id', 'left') + right = _child_folder('right-id', 'right') + left['meta']['typeHierarchy'] = {'salmon': 'fish'} + right['meta']['typeHierarchy'] = {'trout': 'fish'} + folder_cls.return_value.load.side_effect = lambda folder_id, **_kwargs: { + 'left-id': left, + 'right-id': right, + }[folder_id] + valid_images_mock.return_value = [MagicMock(), MagicMock()] + + crud_dataset.create_multicam({'login': 'tester'}, parent, _stereo_data()) + + saved_meta = _saved_parent_meta(folder_cls) + assert saved_meta['typeHierarchy'] == {'salmon': 'fish', 'trout': 'fish'} + assert 'typeHierarchy' not in left['meta'] + assert 'typeHierarchy' not in right['meta'] + + +@patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.valid_images') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_create_multicam_keeps_first_hierarchy_and_warns_for_later_conflict( + _verify, valid_images_mock, folder_cls, _aux +): + parent = _dataset_parent() + parent['meta'] = {} + left = _child_folder('left-id', 'left') + right = _child_folder('right-id', 'right') + left['meta']['typeHierarchy'] = {'salmon': 'fish'} + right['meta']['typeHierarchy'] = {'salmon': 'mammal'} + folder_cls.return_value.load.side_effect = lambda folder_id, **_kwargs: { + 'left-id': left, + 'right-id': right, + }[folder_id] + valid_images_mock.return_value = [MagicMock(), MagicMock()] + + result = crud_dataset.create_multicam({'login': 'tester'}, parent, _stereo_data()) + + assert result['meta']['typeHierarchy'] == {'salmon': 'fish'} + assert result['importWarnings'] == [ + 'Camera "right" type hierarchy was skipped: conflicting parents for "salmon": ' + '"fish" and "mammal"' + ] + assert 'typeHierarchy' not in left['meta'] + assert 'typeHierarchy' not in right['meta'] + saved_meta = _saved_parent_meta(folder_cls) + assert saved_meta['typeHierarchy'] == {'salmon': 'fish'} + + +@patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') +@patch('dive_server.crud_dataset.Folder') +@patch('dive_server.crud_dataset.crud.valid_images') +@patch('dive_server.crud_dataset.crud.verify_dataset') +def test_create_multicam_keeps_camera_hierarchies_when_late_validation_fails( + _verify, valid_images_mock, folder_cls, _aux +): + parent = _dataset_parent() + parent['meta'] = {} + left = _child_folder('left-id', 'left') + right = _child_folder('right-id', 'right') + left['meta']['typeHierarchy'] = {'salmon': 'fish'} + right['meta']['typeHierarchy'] = {'trout': 'fish'} + folder_cls.return_value.load.side_effect = lambda folder_id, **_kwargs: { + 'left-id': left, + 'right-id': right, + }[folder_id] + valid_images_mock.return_value = [MagicMock(), MagicMock()] + data = _stereo_data() + data['subType'] = 'multicam' + data['calibrationFileId'] = 'cal-id' + + with pytest.raises(RestException, match='Calibration is only supported for stereo datasets'): + crud_dataset.create_multicam({'login': 'tester'}, parent, data) + + assert left['meta']['typeHierarchy'] == {'salmon': 'fish'} + assert right['meta']['typeHierarchy'] == {'trout': 'fish'} + + @patch('dive_server.crud_dataset.Item') @patch('dive_server.crud_dataset.crud.get_or_create_auxiliary_folder') @patch('dive_server.crud_dataset.crud.valid_images') diff --git a/server/tests/test_deserialize_kwcoco_json.py b/server/tests/test_deserialize_kwcoco_json.py index 396d29526..fb218aa09 100644 --- a/server/tests/test_deserialize_kwcoco_json.py +++ b/server/tests/test_deserialize_kwcoco_json.py @@ -1,10 +1,15 @@ import json +from pathlib import Path from typing import Dict, List, Tuple import pytest from dive_utils.serializers import kwcoco +KWCOCO_PROFILE = json.loads( + (Path(__file__).parents[2] / 'testutils/kwcoco/import-profile.json').read_text() +) + test_tuple: List[Tuple[dict, dict, dict]] = [ ( # test if coco native is handled properly @@ -727,6 +732,37 @@ def test_export_dive_as_coco_single_dataset(): assert "dive_notes" in coco["info"]["dive_extensions"] +def test_export_dive_as_coco_preserves_pairs_and_category_hierarchy_roundtrip(): + profile = KWCOCO_PROFILE['exportRoundTrip'] + exported = kwcoco.export_dive_as_coco( + profile['tracks'], + {int(frame): name for frame, name in profile['imageFilenames'].items()}, + dataset_name=profile['datasetName'], + typeHierarchy=profile['typeHierarchy'], + ) + categories = {category['name']: category for category in exported['categories']} + assert list(categories) == profile['expectedCategoryNames'] + assert { + name: category['supercategory'] + for name, category in categories.items() + if 'supercategory' in category + } == profile['expectedParents'] + annotation = exported['annotations'][0] + assert annotation['track_id'] == profile['tracks'][0]['id'] + assert annotation['category_id'] == categories['leaf']['id'] + assert annotation['score'] == 0.75 + assert annotation['prob'] == profile['expectedProb'] + assert annotation['dive_confidence_pairs'] == profile['expectedPairs'] + assert 'dive_confidence_pairs' in exported['info']['dive_extensions'] + + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(exported) + track_id = str(profile['tracks'][0]['id']) + assert converted['tracks'][track_id]['confidencePairs'] == [ + tuple(pair) for pair in profile['expectedPairs'] + ] + assert warnings == [] + + # --- datasetInfo passthrough --- DATASET_INFO = { @@ -936,3 +972,200 @@ def test_import_polygon_and_rle_segmentation(): assert rle_track["features"][0]["bounds"] == [400, 200, 600, 260] assert "geometry" not in rle_track["features"][0] assert len(warnings) == 1 + + +def _classification_coco(categories, annotations): + return { + 'images': [ + {'id': 1, 'file_name': 'frame_1.jpg', 'frame_index': 1}, + {'id': 2, 'file_name': 'frame_2.jpg', 'frame_index': 2}, + ], + 'annotations': annotations, + 'categories': categories, + } + + +def _classification_annotation(annotation_id, image_id=1, **extra): + return { + 'id': annotation_id, + 'image_id': image_id, + 'category_id': 1, + 'track_id': 9, + 'bbox': [1, 2, 3, 4], + **extra, + } + + +def test_prob_uses_raw_category_order_and_preserves_unnamed_slots(): + coco = _classification_coco( + [{'id': 10, 'name': 'fish'}, {'id': 1}, {'id': 4, 'name': 'shark'}], + [_classification_annotation(1, category_id=10, prob=[0.2, 0.9, 0.1])], + ) + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(coco) + assert converted['tracks']['9']['confidencePairs'] == [('fish', 0.2), ('shark', 0.1)] + assert warnings == [] + + +def test_unnamed_primary_category_falls_back_to_unknown(): + coco = _classification_coco( + [{'id': 1}, {'id': 2, 'name': 'fish'}], + [_classification_annotation(1, category_id=1)], + ) + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(coco) + assert converted['tracks']['9']['confidencePairs'] == [('unknown', 1.0)] + assert warnings == [] + + +def test_prob_prunes_and_warns_once_for_mismatch_or_duplicate_names(): + categories = [{'id': index, 'name': f'class_{index}'} for index in range(12)] + annotations = [ + _classification_annotation( + 1, + track_id=1, + prob=[0.5 - index * 0.01 for index in range(12)], + ), + _classification_annotation(2, track_id=2, prob=[0.1]), + _classification_annotation(3, track_id=3, prob=[0.2]), + ] + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes( + _classification_coco(categories, annotations) + ) + assert len(converted['tracks']['1']['confidencePairs']) == 10 + assert warnings == [kwcoco.PROB_LENGTH_MISMATCH_WARNING] + + duplicate = _classification_coco( + [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'fish'}], + [_classification_annotation(4, prob=[0.1, 0.9])], + ) + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(duplicate) + assert converted['tracks']['9']['confidencePairs'] == [('fish', 1.0)] + assert warnings == [kwcoco.PROB_DUPLICATE_CATEGORY_WARNING] + + +def test_dive_confidence_pairs_prefer_exact_sparse_zero_membership(): + coco = _classification_coco( + [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'shark'}], + [ + _classification_annotation( + 1, + prob=[0.1, 0.9], + dive_confidence_pairs=[['shark', 0.0], ['fish', 0.25]], + ) + ], + ) + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(coco) + assert converted['tracks']['9']['confidencePairs'] == [('shark', 0.0), ('fish', 0.25)] + assert warnings == [] + + +@pytest.mark.parametrize( + 'value', + [ + [], + 'not a pair list', + [['fish']], + [['fish', 0.2], ['fish', 0.3]], + [['fish', float('nan')]], + [['fish', 1.1]], + ], +) +def test_malformed_dive_confidence_pairs_warns_once_and_falls_back(value): + coco = _classification_coco( + [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'shark'}], + [ + _classification_annotation(1, prob=[0.2, 0.8], dive_confidence_pairs=value), + _classification_annotation(2, prob=[0.2, 0.8], dive_confidence_pairs=value), + ], + ) + + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(coco) + + assert converted['tracks']['9']['confidencePairs'] == [('shark', 0.8), ('fish', 0.2)] + assert warnings == [kwcoco.DIVE_CONFIDENCE_PAIRS_WARNING] + + +def test_highest_frame_confidence_wins_independent_of_source_order(): + categories = [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'shark'}] + annotations = [ + _classification_annotation(1, image_id=2, prob=[0.2, 0.8]), + _classification_annotation(2, image_id=1, prob=[0.9, 0.1]), + ] + converted, _, _, _ = kwcoco.load_coco_as_tracks_and_attributes( + _classification_coco(categories, annotations) + ) + assert converted['tracks']['9']['confidencePairs'] == [('shark', 0.8), ('fish', 0.2)] + + +def test_same_highest_frame_uses_greater_annotation_id_independent_of_source_order(): + categories = [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'shark'}] + annotations = [ + _classification_annotation(2, prob=[0.9, 0.1]), + _classification_annotation(1, prob=[0.2, 0.8]), + ] + document = _classification_coco(categories, annotations) + + converted, _, _, _ = kwcoco.load_coco_as_tracks_and_attributes(document) + reordered, _, _, _ = kwcoco.load_coco_as_tracks_and_attributes( + {**document, 'annotations': list(reversed(annotations))} + ) + + expected = [('fish', 0.9), ('shark', 0.1)] + assert converted['tracks']['9']['confidencePairs'] == expected + assert reordered['tracks']['9']['confidencePairs'] == expected + + +def test_supercategory_extraction_handles_roots_duplicates_and_multiple_parents(): + hierarchy, warnings = kwcoco.type_hierarchy_from_categories( + { + 'categories': [ + {'id': 1, 'name': 'root', 'supercategory': 'root'}, + {'id': 2, 'name': 'leaf', 'supercategory': 'root', 'parents': ['root', 'other']}, + {'id': 3, 'name': 'external', 'supercategory': 'outside'}, + {'id': 4}, + ] + } + ) + assert hierarchy == {'leaf': 'root', 'external': 'outside'} + assert warnings == [ + kwcoco.SUPERCATEGORY_MULTI_PARENT_WARNING, + kwcoco.CATEGORY_MISSING_NAME_WARNING, + ] + + hierarchy, warnings = kwcoco.type_hierarchy_from_categories( + {'categories': [{'id': 1, 'name': 'fish'}, {'id': 2, 'name': 'fish'}]} + ) + assert hierarchy is None + assert warnings == [kwcoco.SUPERCATEGORY_DUPLICATE_CATEGORY_WARNING] + + +def test_shared_exact_vector_and_hierarchy_import_profile(): + profile = KWCOCO_PROFILE['highestFrameExact'] + + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(profile['document']) + hierarchy, hierarchy_warnings = kwcoco.type_hierarchy_from_categories(profile['document']) + + pairs = converted['tracks'][str(profile['trackId'])]['confidencePairs'] + assert [list(pair) for pair in pairs] == profile['expectedPairs'] + assert hierarchy == profile['expectedHierarchy'] + assert warnings == [] + assert hierarchy_warnings == [] + + +def test_shared_missing_frame_index_profile(): + profile = KWCOCO_PROFILE['missingFrameIndexExact'] + + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(profile['document']) + + pairs = converted['tracks'][str(profile['trackId'])]['confidencePairs'] + assert [list(pair) for pair in pairs] == profile['expectedPairs'] + assert warnings == [] + + +def test_shared_empty_dive_confidence_pairs_profile(): + profile = KWCOCO_PROFILE['emptyDiveConfidencePairs'] + + converted, _, warnings, _ = kwcoco.load_coco_as_tracks_and_attributes(profile['document']) + + pairs = converted['tracks'][str(profile['trackId'])]['confidencePairs'] + assert [list(pair) for pair in pairs] == profile['expectedPairs'] + assert warnings == [kwcoco.DIVE_CONFIDENCE_PAIRS_WARNING] diff --git a/server/tests/test_hierarchy_ingestion_routes.py b/server/tests/test_hierarchy_ingestion_routes.py index 286f3e981..6ded19e3c 100644 --- a/server/tests/test_hierarchy_ingestion_routes.py +++ b/server/tests/test_hierarchy_ingestion_routes.py @@ -6,6 +6,7 @@ from dive_server import crud_rpc from dive_utils import constants +from dive_utils.serializers import kwcoco from dive_utils.type_hierarchy import TypeHierarchyError @@ -94,7 +95,7 @@ def update_metadata(target, payload, _verify, hierarchy_mode='save'): monkeypatch.setattr(crud_rpc.crud_dataset, 'update_metadata', update_metadata) plan = { 'parent': None, - 'hierarchy_instructions': [(True, incoming)], + 'hierarchy_instructions': [crud_rpc.HierarchyInstruction(True, incoming)], 'additive': additive, 'staged_meta': {}, 'staged_parent_meta': {}, @@ -135,7 +136,7 @@ def remove_copy(target): monkeypatch.setattr(crud_rpc.crud_dataset, 'remove_camera_type_hierarchy', remove_copy) plan = { 'parent': parent, - 'hierarchy_instructions': [(True, {'salmon': 'fish'})], + 'hierarchy_instructions': [crud_rpc.HierarchyInstruction(True, {'salmon': 'fish'})], 'additive': True, 'staged_meta': {'imageEnhancements': {'brightness': 1.1}}, 'staged_parent_meta': {'confidenceFilters': {'default': 0.4}}, @@ -222,26 +223,6 @@ def test_camera_without_stored_hierarchy_import_warns_nothing(monkeypatch): assert 'warnings' not in plan -def test_camera_configuration_conflict_is_rejected_before_writes(monkeypatch): - parent = {'_id': 'parent', 'meta': {'typeHierarchy': {'salmon': 'fish'}}} - camera = {'_id': 'camera', 'meta': {}} - update_metadata = MagicMock() - monkeypatch.setattr(crud_rpc, '_fresh_folder_snapshot', lambda target: target) - monkeypatch.setattr(crud_rpc.crud_dataset, 'update_metadata', update_metadata) - plan = { - 'parent': parent, - 'hierarchy_instructions': [(True, {'salmon': 'mammal'})], - 'additive': True, - 'staged_meta': {}, - 'staged_parent_meta': {}, - 'applied': False, - } - - with pytest.raises(RestException, match='conflicting parents for "salmon"'): - crud_rpc._apply_configuration_imports(camera, plan) - update_metadata.assert_not_called() - - def test_camera_stored_hierarchy_is_promoted_without_new_configuration(monkeypatch): parent = {'_id': 'parent', 'meta': {'type': constants.MultiType}} camera = { @@ -276,6 +257,26 @@ def update_metadata(target, payload, _verify, hierarchy_mode='save'): assert 'typeHierarchy' not in camera['meta'] +def test_camera_configuration_conflict_is_rejected_before_writes(monkeypatch): + parent = {'_id': 'parent', 'meta': {'typeHierarchy': {'salmon': 'fish'}}} + camera = {'_id': 'camera', 'meta': {}} + update_metadata = MagicMock() + monkeypatch.setattr(crud_rpc, '_fresh_folder_snapshot', lambda target: target) + monkeypatch.setattr(crud_rpc.crud_dataset, 'update_metadata', update_metadata) + plan = { + 'parent': parent, + 'hierarchy_instructions': [crud_rpc.HierarchyInstruction(True, {'salmon': 'mammal'})], + 'additive': True, + 'staged_meta': {}, + 'staged_parent_meta': {}, + 'applied': False, + } + + with pytest.raises(RestException, match='conflicting parents for "salmon"'): + crud_rpc._apply_configuration_imports(camera, plan) + update_metadata.assert_not_called() + + def test_conflicting_camera_hierarchy_is_skipped_before_incoming_configuration(monkeypatch): parent = {'_id': 'parent', 'meta': {'typeHierarchy': {'salmon': 'fish'}}} camera = { @@ -297,7 +298,7 @@ def update_metadata(target, payload, _verify, hierarchy_mode='save'): ) plan = { 'parent': parent, - 'hierarchy_instructions': [(True, {'shark': 'fish'})], + 'hierarchy_instructions': [crud_rpc.HierarchyInstruction(True, {'shark': 'fish'})], 'additive': True, 'staged_meta': {}, 'staged_parent_meta': {}, @@ -344,6 +345,44 @@ def test_failed_parent_save_leaves_camera_hierarchy_for_retry(monkeypatch): remove_copy.assert_not_called() +def test_soft_coco_hierarchy_conflict_warns_and_preserves_the_candidate(): + write, warnings = crud_rpc._resolve_configuration_hierarchy( + {'shark': 'animal'}, + [ + crud_rpc.HierarchyInstruction( + True, + {'shark': 'fish'}, + kwcoco.SUPERCATEGORY_INVALID_WARNING, + ) + ], + additive=False, + ) + + assert write == {'action': 'none'} + assert warnings == [ + kwcoco.SUPERCATEGORY_INVALID_WARNING.format( + reason='conflicting parents for "shark": "animal" and "fish"' + ) + ] + + +def test_first_soft_coco_hierarchy_initializes_an_absent_candidate(): + write, warnings = crud_rpc._resolve_configuration_hierarchy( + None, + [ + crud_rpc.HierarchyInstruction( + True, + {'shark': 'fish'}, + kwcoco.SUPERCATEGORY_INVALID_WARNING, + ) + ], + additive=False, + ) + + assert write == {'action': 'set', 'hierarchy': {'shark': 'fish'}} + assert warnings == [] + + def test_postprocess_delegates_without_private_preflight_protocol(monkeypatch): expected = {'folder': {'_id': 'dataset'}, 'job_ids': []} postprocess = MagicMock(return_value=expected) diff --git a/server/tests/test_multicam_export_clone.py b/server/tests/test_multicam_export_clone.py index 6c644a4ee..14cfc4c40 100644 --- a/server/tests/test_multicam_export_clone.py +++ b/server/tests/test_multicam_export_clone.py @@ -725,3 +725,28 @@ def footer(self): assert 'stereo-dataset/left/annotations.viame.csv' in paths assert 'stereo-dataset/right/annotations.viame.csv' in paths assert csv_gen_mock.call_count == 2 + + +@patch('dive_server.crud_dataset.ziputil.ZipGenerator') +def test_export_multicam_annotations_preflights_invalid_coco_hierarchy(zip_gen_cls, monkeypatch): + parent = _multi_parent_folder() + hierarchy_error = RestException( + 'Type hierarchy is invalid: cycle detected. No configuration file was exported.' + ) + monkeypatch.setattr( + crud_dataset, + 'type_hierarchy_for_export', + MagicMock(side_effect=hierarchy_error), + ) + + with pytest.raises(RestException, match='cycle detected'): + crud_dataset.export_multicam_annotations_zipstream( + parent, + {'login': 'tester'}, + 'coco_json', + False, + None, + None, + ) + + zip_gen_cls.assert_not_called() diff --git a/server/tests/test_update_metadata.py b/server/tests/test_update_metadata.py index 4470406d6..8d96fdd00 100644 --- a/server/tests/test_update_metadata.py +++ b/server/tests/test_update_metadata.py @@ -316,7 +316,7 @@ def test_get_data_by_type_classifies_presence_only_type_hierarchy_as_config(file assert warnings is None assert result['type'] == crud.FileType.DIVE_CONF - assert result['meta']['typeHierarchy'] is None + assert result['meta']['typeHierarchy'] == hierarchy def test_metadata_mutable_does_not_classify_unrelated_json_as_config(): diff --git a/testutils/kwcoco/import-profile.json b/testutils/kwcoco/import-profile.json new file mode 100644 index 000000000..e071b91b2 --- /dev/null +++ b/testutils/kwcoco/import-profile.json @@ -0,0 +1,139 @@ +{ + "highestFrameExact": { + "document": { + "info": { + "dive_extensions": ["dive_confidence_pairs"] + }, + "images": [ + { "id": 20, "file_name": "frame_000009.jpg", "frame_index": 9 }, + { "id": 10, "file_name": "frame_000002.jpg", "frame_index": 2 } + ], + "annotations": [ + { + "id": 100, + "image_id": 20, + "category_id": 7, + "track_id": 17, + "bbox": [10, 20, 30, 40], + "score": 0.8, + "prob": [0, 0.8, 0.2], + "dive_confidence_pairs": [["shark", 0.8], ["fish", 0], ["rock", 0.2]] + }, + { + "id": 101, + "image_id": 10, + "category_id": 5, + "track_id": 17, + "bbox": [11, 21, 30, 40], + "score": 0.9, + "prob": [0.9, 0.1, 0], + "dive_confidence_pairs": [["fish", 0.9], ["shark", 0.1]] + } + ], + "categories": [ + { "id": 5, "name": "fish" }, + { "id": 7, "name": "shark", "supercategory": "fish" }, + { "id": 11, "name": "rock" } + ] + }, + "trackId": 17, + "expectedPairs": [["shark", 0.8], ["fish", 0], ["rock", 0.2]], + "expectedHierarchy": { "shark": "fish" } + }, + "missingFrameIndexExact": { + "document": { + "info": { + "dive_extensions": ["dive_confidence_pairs"] + }, + "images": [ + { "id": 1, "file_name": "Z1.jpg" }, + { "id": 2, "file_name": "a10.jpg" } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "track_id": 9, + "bbox": [0, 0, 10, 10], + "score": 0.4, + "dive_confidence_pairs": [["early", 0.4], ["explicit-zero", 0]] + }, + { + "id": 2, + "image_id": 2, + "category_id": 2, + "track_id": 9, + "bbox": [1, 1, 10, 10], + "score": 0.8, + "dive_confidence_pairs": [["late", 0.8]] + } + ], + "categories": [ + { "id": 1, "name": "early" }, + { "id": 2, "name": "late" } + ] + }, + "trackId": 9, + "expectedPairs": [["late", 0.8]] + }, + "emptyDiveConfidencePairs": { + "document": { + "info": { + "dive_extensions": ["dive_confidence_pairs"] + }, + "images": [ + { "id": 1, "file_name": "frame_000000.jpg", "frame_index": 0 } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "track_id": 7, + "bbox": [0, 0, 10, 10], + "score": 0.3, + "prob": [0.2, 0.8], + "dive_confidence_pairs": [] + } + ], + "categories": [ + { "id": 1, "name": "fish" }, + { "id": 2, "name": "shark" } + ] + }, + "trackId": 7, + "expectedPairs": [["shark", 0.8], ["fish", 0.2]] + }, + "exportRoundTrip": { + "datasetName": "shared-roundtrip", + "typeHierarchy": { + "leaf": "parent", + "parent": "root", + "unused": "root" + }, + "imageFilenames": { + "0": "frame_000000.jpg" + }, + "tracks": [ + { + "id": 42, + "begin": 0, + "end": 0, + "confidencePairs": [["parent", 0], ["leaf", 0.75], ["other", 0.25]], + "attributes": {}, + "features": [ + { "frame": 0, "bounds": [10, 20, 30, 60] } + ] + } + ], + "expectedCategoryNames": ["parent", "leaf", "other", "unused", "root"], + "expectedParents": { + "leaf": "parent", + "parent": "root", + "unused": "root" + }, + "expectedProb": [0, 0.75, 0.25, 0, 0], + "expectedPairs": [["parent", 0], ["leaf", 0.75], ["other", 0.25]] + } +}