From 48a2824295fb211f27c83dd96069549cfd66ef5a Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 14 Aug 2026 17:14:17 -0400 Subject: [PATCH 1/2] feat: warn about Zarr layouts that make viewing awkward Check the dataset layout client-side from metadata that is already parsed, and surface the result through the MetadataHint notifications used for broken metadata. All three checks are advisory - nothing is withheld, and viewer links are untouched. Resolution levels: a multiscales group declaring a single dataset provides no downsampled data, so every zoom reads full-resolution chunks. This was the root cause of the production incident behind these checks. Keyed on the dataset count rather than the number of shapes, because a plain zarr array also has exactly one shape while claiming nothing about being multiscale. Chunk size: chunks past the browser's cache entry limit are re-fetched on every access. Sizes are computed from shape and dtype, so they are logical: exact for a raw array, an upper bound for a compressed one. The limit is 10 MB when no compressor is present and 32 MB when one is, since the compression ratio is a property of the pixels and cannot be derived from metadata. Unknown or unfetched codec info counts as compressed, so uncertainty costs a missed warning rather than a false one. Codec classification reuses capability-manifest's classifyCodec, plus a walk over the pipeline it does not do itself: a sharded v3 array lists only the structural sharding_indexed codec at the top level and carries the real compressor in its configuration. Plain arrays now carry codec info too - the v3 branch reads it from the zarr.json it already parsed, the v2 branch fetches .zarray as the group path does. Axis order: OME-Zarr requires T, C, Z, Y, X order, and many tools take the last two axes to be the image plane. A dataset ordered C, X, Y, Z therefore renders as a Y-Z cross-section in those tools while Neuroglancer, which reads the axis names, looks fine - a confusing failure worth naming explicitly. Datasets using custom axis names are left alone, since their intent cannot be judged. No thumbnail check: ome-zarr.js already refuses when the lowest level exceeds its own limit, and modelling its behaviour a second time proved unreliable - it renders from the level nearest the requested size rather than the smallest, and reads a single plane rather than the whole level. Co-Authored-By: Claude Opus 5 (1M context) --- frontend/src/__tests__/mocks/omezarrHelper.ts | 3 +- .../unitTests/datasetWarnings.test.ts | 197 ++++++++++++++++++ .../components/ui/BrowsePage/MetadataHint.tsx | 22 ++ .../components/ui/BrowsePage/ZarrPreview.tsx | 16 +- frontend/src/omezarr-helper.ts | 169 +++++++++++++++ frontend/src/queries/zarrQueries.ts | 23 +- 6 files changed, 426 insertions(+), 4 deletions(-) create mode 100644 frontend/src/__tests__/unitTests/datasetWarnings.test.ts diff --git a/frontend/src/__tests__/mocks/omezarrHelper.ts b/frontend/src/__tests__/mocks/omezarrHelper.ts index 92422468..ce7addc6 100644 --- a/frontend/src/__tests__/mocks/omezarrHelper.ts +++ b/frontend/src/__tests__/mocks/omezarrHelper.ts @@ -48,5 +48,6 @@ export const omezarrHelperMock = { generateNeuroglancerStateForOmeZarr: vi.fn(() => 'mock-state-ome-zarr'), determineLayerType: vi.fn(async () => 'image'), translateUnitToNeuroglancer: vi.fn((unit: string) => unit), - getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]) + getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]), + getDatasetWarnings: vi.fn(() => []) }; diff --git a/frontend/src/__tests__/unitTests/datasetWarnings.test.ts b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts new file mode 100644 index 00000000..a86a25d8 --- /dev/null +++ b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect } from 'vitest'; +import { getDatasetWarnings } from '@/omezarr-helper'; +import type { Metadata } from '@/omezarr-helper'; + +// Minimal stand-in for the parts of Metadata the checks read. Codec info is +// left out by default, which the chunk check treats as compressed. +const createMetadata = ( + chunks: number[], + dtype = 'uint16', + extra: Partial = {}, + shape: number[] = [8, 8, 8] +): Metadata => + ({ + arr: { chunks, dtype, shape }, + ...extra + }) as unknown as Metadata; + +const axes = (...names: string[]): Partial => ({ + multiscales: [ + { axes: names.map(name => ({ name, type: 'space' })), datasets: [{}, {}] } + ] as unknown as Metadata['multiscales'] +}); + +const levels = (count: number): Partial => ({ + multiscales: [ + { datasets: Array.from({ length: count }, () => ({})) } + ] as unknown as Metadata['multiscales'] +}); + +// zstd nested inside a sharding_indexed pipeline, as a sharded v3 array stores it. +const SHARDED_ZSTD: Partial = { + codecs: [ + { + name: 'sharding_indexed', + configuration: { codecs: [{ name: 'bytes' }, { name: 'zstd' }] } + } + ] +}; +const UNCOMPRESSED_V3: Partial = { + codecs: [{ name: 'bytes' }, { name: 'crc32c' }] +}; + +describe('getDatasetWarnings: chunk size', () => { + it('says nothing about reasonable chunks', () => { + expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); + }); + + it('does not warn about a compressed 16 MB chunk', () => { + // raw/s2: 16 MB inner chunks that zstd takes to ~12 MB on disk. + expect( + getDatasetWarnings( + createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('holds an uncompressed array to the stricter limit', () => { + // The same 16 MB chunks, but stored raw, so 16 MB is what transfers. + for (const raw of [UNCOMPRESSED_V3, { compressor: null }]) { + expect( + getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8', raw)) + ).toEqual([ + { case: 'zarr-large-chunks', size: '16 MB', compressed: false } + ]); + } + }); + + it('finds a compressor nested inside a sharding codec', () => { + // sharding_indexed is structural, so a flat scan would call this + // uncompressed and warn at 16 MB. + expect( + getDatasetWarnings( + createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('assumes compressed when codec metadata was never fetched', () => { + // Unknown lands on the permissive limit: a missed warning beats a false one. + expect( + getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8')) + ).toEqual([]); + }); + + it('warns above the compressed limit', () => { + // seed151 img: 128 MB chunks. + expect( + getDatasetWarnings(createMetadata([256, 256, 256, 8], 'uint8')) + ).toEqual([ + { case: 'zarr-large-chunks', size: '128 MB', compressed: true } + ]); + }); + + it('accounts for the dtype width', () => { + expect(getDatasetWarnings(createMetadata([256, 256, 256]))).toEqual([]); + expect( + getDatasetWarnings(createMetadata([256, 256, 256], 'float64')) + ).toHaveLength(1); + }); +}); + +describe('getDatasetWarnings: resolution levels', () => { + const BIG = [3000, 3000, 1350, 8]; // 91 GB of uint8, the seed151 img extent + + it('warns when multiscales declares a single level for a large image', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(1), BIG)) + ).toEqual([{ case: 'zarr-single-level', size: '91 GB' }]); + }); + + it('says nothing when the pyramid has levels', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(5), BIG)) + ).toEqual([]); + }); + + it('says nothing about a small single-level image', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint8', levels(1), [256, 256, 256]) + ) + ).toEqual([]); + }); + + it('never fires on a plain array, however large', () => { + // The bug that made this warn on raw/s2: a plain array also has one shape, + // but it declares no multiscales and so claims nothing. + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', {}, BIG)) + ).toEqual([]); + }); +}); + +describe('getDatasetWarnings: axis order', () => { + it('accepts spec order', () => { + for (const names of [ + ['t', 'c', 'z', 'y', 'x'], + ['c', 'z', 'y', 'x'], + ['z', 'y', 'x'], + ['y', 'x'] + ]) { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes(...names)) + ) + ).toEqual([]); + } + }); + + it('warns about the seed151 c,x,y,z order', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64, 8], 'uint8', axes('c', 'x', 'y', 'z')) + ) + ).toEqual([ + { + case: 'zarr-axis-order', + axisOrder: 'C, X, Y, Z', + expectedOrder: 'C, Z, Y, X' + } + ]); + }); + + it('warns when the channel axis trails the spatial axes', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64, 8], 'uint8', axes('x', 'y', 'z', 'c')) + ) + ).toEqual([ + { + case: 'zarr-axis-order', + axisOrder: 'X, Y, Z, C', + expectedOrder: 'C, Z, Y, X' + } + ]); + }); + + it('is case insensitive', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes('Z', 'Y', 'X')) + ) + ).toEqual([]); + }); + + it('stays quiet about custom axes it cannot judge', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint16', axes('c', 'angle', 'y', 'x')) + ) + ).toEqual([]); + }); + + it('stays quiet about a plain array with no axes', () => { + expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); + }); +}); diff --git a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx index 13fadf2f..0650026f 100644 --- a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx +++ b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx @@ -12,6 +12,10 @@ type MetadataHintVariant = | { case: 'zarr-v2-no-multiscales' } | { case: 'zarr-v3-no-multiscales' } | { case: 'zarr-query-error'; errorMessage?: string } + // Zarr - metadata is valid, but the layout will make viewing awkward + | { case: 'zarr-single-level'; size: string } + | { case: 'zarr-large-chunks'; size: string; compressed: boolean } + | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string } // N5 - query never fired | { case: 'n5-has-s0-no-attrs' } | { case: 'n5-has-attrs-no-s0' } @@ -72,6 +76,24 @@ function getHintConfig(variant: MetadataHintVariant): HintConfig { ? `Could not read Zarr metadata. ${variant.errorMessage}` : 'Could not read Zarr metadata.' }; + case 'zarr-single-level': + return { + kind: 'warning', + title: 'Only one resolution level', + description: `This dataset declares multiscales but only supplies a single level, for ${variant.size} of data. Viewers must read full-resolution at every zoom level, so viewing the whole image is far more expensive than it needs to be. Generating a multiscale pyramid fixes this.` + }; + case 'zarr-axis-order': + return { + kind: 'warning', + title: 'Axes are not in the order OME-Zarr specifies', + description: `The axes are ordered ${variant.axisOrder}, but the spec requires T, C, Z, Y, X order - here that would be ${variant.expectedOrder}. Many tools take the last two axes to be the image plane, so they will show a cross-section rather than the expected view. Rewriting the dataset with the axes in spec order avoids this.` + }; + case 'zarr-large-chunks': + return { + kind: 'warning', + title: 'Chunks may be too large for efficient viewing', + description: `This dataset uses ${variant.size} chunks ${variant.compressed ? '(before compression)' : '(without compression)'}. Chunk files larger than the browser's cache limit are re-downloaded on every access, which makes viewing slow. A final chunk size of 1-10 MB works best.` + }; case 'n5-has-s0-no-attrs': logger.info( 'This folder has a .n5 extension but does not contain an attributes.json file required for N5 metadata preview.' diff --git a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx index fac230b7..d7d0aa8f 100644 --- a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx +++ b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx @@ -6,13 +6,14 @@ import zarrLogo from '@/assets/zarr.jpg'; import ZarrMetadataTable from '@/components/ui/BrowsePage/ZarrMetadataTable'; import DataLinkDialog from '@/components/ui/Dialogs/DataLink'; import DataToolLinks from './DataToolLinks'; +import MetadataHint from './MetadataHint'; import type { OpenWithToolUrls, ZarrMetadata, PendingToolKey } from '@/hooks/useZarrMetadata'; import useDataToolLinks from '@/hooks/useDataToolLinks'; -import { Metadata } from '@/omezarr-helper'; +import { Metadata, getDatasetWarnings } from '@/omezarr-helper'; type ZarrPreviewProps = { readonly fspName: string; @@ -41,6 +42,12 @@ export default function ZarrPreview({ const [showDataLinkDialog, setShowDataLinkDialog] = useState(false); const [pendingToolKey, setPendingToolKey] = useState(null); + const metadata = zarrMetadataQuery.data?.metadata; + const warnings = + metadata && 'arr' in metadata + ? getDatasetWarnings(metadata as Metadata) + : []; + const { handleToolClick, handleDialogConfirm, @@ -55,6 +62,13 @@ export default function ZarrPreview({ return (
+ {warnings.length > 0 ? ( +
+ {warnings.map(warning => ( + + ))} +
+ ) : null}
diff --git a/frontend/src/omezarr-helper.ts b/frontend/src/omezarr-helper.ts index c34369b2..a135c310 100644 --- a/frontend/src/omezarr-helper.ts +++ b/frontend/src/omezarr-helper.ts @@ -1,6 +1,8 @@ import { default as log } from '@/logger'; +import { formatFileSize } from '@/utils'; import * as zarr from 'zarrita'; import * as omezarr from 'ome-zarr.js'; +import { classifyCodec } from '@bioimagetools/capability-manifest'; import type { OmeZarrMetadata, MultiscaleMetadata, @@ -22,6 +24,173 @@ export type Metadata = OmeZarrMetadata & { zarrVersion: 2 | 3; }; +/** + * Something about the dataset's layout that will make it awkward to view. + * Purely advisory - nothing is withheld on account of these. + */ +export type DatasetWarning = + | { case: 'zarr-single-level'; size: string } + | { case: 'zarr-large-chunks'; size: string; compressed: boolean } + | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string }; + +/** + * Chunks above this defeat the browser cache. Applies when the array is stored + * without compression, so the size we compute is the size that transfers. + */ +export const MAX_CHUNK_BYTES = 10 * 1024 ** 2; +/** + * The same limit for compressed arrays, where all we can compute is the logical + * (uncompressed) extent and the real transfer is some unknowable fraction of it. + * Set high enough that a typical ratio cannot push a healthy dataset over. + */ +export const MAX_LOGICAL_CHUNK_BYTES = 32 * 1024 ** 2; +/** + * Below this, a single-level image is small enough that the missing pyramid + * costs nothing worth mentioning. + */ +export const MAX_SINGLE_LEVEL_BYTES = 1024 ** 3; + +/** The axis order OME-Zarr requires, minus any axes the dataset omits. */ +const CANONICAL_AXIS_ORDER = ['t', 'c', 'z', 'y', 'x']; + +/** + * Whether the array's chunks are compressed on disk. + * + * A codec pipeline can nest: a sharded v3 array lists only `sharding_indexed` + * at the top level and carries the real compressor in its configuration, so the + * pipeline has to be walked rather than scanned. Anything `classifyCodec` does + * not recognize counts as compression, and metadata we never fetched counts as + * compression too - both keep us on the permissive threshold, where the cost of + * being wrong is a missed warning instead of a false one. + */ +function hasCompressionCodec(codecs: NonNullable): boolean { + return codecs.some(codec => { + const nested = codec.configuration?.codecs; + if (Array.isArray(nested) && hasCompressionCodec(nested)) { + return true; + } + return classifyCodec(codec.name) !== 'structural'; + }); +} + +function isStoredCompressed(metadata: Metadata): boolean { + if (metadata.codecs) { + return hasCompressionCodec(metadata.codecs); + } + if (metadata.compressor !== undefined) { + return metadata.compressor !== null; + } + return true; +} + +/** + * Bytes per element for a zarrita dtype. Only numeric dtypes are sized; bool, + * string and object dtypes fall back to 1, which under-estimates rather than + * over-warns (they don't occur in imaging data). + */ +function getBytesPerElement(dtype: string): number { + const bits = Number(/^(?:u?int|float)(\d+)$/.exec(dtype)?.[1]); + return Number.isFinite(bits) ? bits / 8 : 1; +} + +function product(dims: number[]): number { + return dims.reduce((total, dim) => total * dim, 1); +} + +/** + * The axis names, lowercased, or null if any of them is not one of the five the + * spec defines. A dataset using custom axes is not something we can judge. + */ +function getCanonicalAxisNames(metadata: Metadata): string[] | null { + const axes = metadata.multiscales?.[0]?.axes ?? metadata.axes; + if (!axes?.length) { + return null; + } + const names = axes.map(axis => axis.name?.toLowerCase()); + return names.every(name => name && CANONICAL_AXIS_ORDER.includes(name)) + ? (names as string[]) + : null; +} + +/** Whether `names` appears in CANONICAL_AXIS_ORDER order, skipping absent axes. */ +function isCanonicallyOrdered(names: string[]): boolean { + let from = 0; + return names.every(name => { + const index = CANONICAL_AXIS_ORDER.indexOf(name, from); + if (index === -1) { + return false; + } + from = index + 1; + return true; + }); +} + +const formatAxes = (names: string[]) => + names.map(name => name.toUpperCase()).join(', '); + +/** + * Flag layout choices that make a dataset awkward to view. + * + * A multiscales group with a single dataset provides no downsampled data, so + * every zoom level reads full-resolution chunks - the root cause of the incident + * that prompted these checks. + * + * Chunks past the browser's cache entry limit are re-fetched on every access. + * Sizes are computed from the shape, so they are logical: exact for an + * uncompressed array and an upper bound for a compressed one, which is why the + * limit depends on whether a compressor is in play. + * + * Axis order matters because plenty of tools take the last two axes to be the + * image plane. OME-Zarr requires t, c, z, y, x order for exactly that reason, + * and a dataset that ignores it renders as a cross-section elsewhere. + */ +export function getDatasetWarnings(metadata: Metadata): DatasetWarning[] { + const { arr } = metadata; + if (!arr) { + return []; + } + + const bytesPerElement = getBytesPerElement(arr.dtype); + const warnings: DatasetWarning[] = []; + + // Declaring multiscales with one dataset is declaring a pyramid and supplying + // none. Keyed on the dataset count rather than the number of shapes, because + // a plain zarr array also has exactly one shape and is not making any such + // claim - `arr` is level 0, so its shape is the full resolution. + const levels = metadata.multiscales?.[0]?.datasets?.length; + const fullResBytes = product(arr.shape) * bytesPerElement; + if (levels === 1 && fullResBytes > MAX_SINGLE_LEVEL_BYTES) { + warnings.push({ + case: 'zarr-single-level', + size: formatFileSize(fullResBytes) + }); + } + + const compressed = isStoredCompressed(metadata); + const chunkBytes = product(arr.chunks) * bytesPerElement; + const chunkLimit = compressed ? MAX_LOGICAL_CHUNK_BYTES : MAX_CHUNK_BYTES; + if (chunkBytes > chunkLimit) { + warnings.push({ + case: 'zarr-large-chunks', + size: formatFileSize(chunkBytes), + compressed + }); + } + + const axisNames = getCanonicalAxisNames(metadata); + if (axisNames && !isCanonicallyOrdered(axisNames)) { + warnings.push({ + case: 'zarr-axis-order', + axisOrder: formatAxes(axisNames), + expectedOrder: formatAxes( + CANONICAL_AXIS_ORDER.filter(name => axisNames.includes(name)) + ) + }); + } + + return warnings; +} + type OmeZarrChannel = { name: string; color: string; diff --git a/frontend/src/queries/zarrQueries.ts b/frontend/src/queries/zarrQueries.ts index a574ffc0..d449a10a 100644 --- a/frontend/src/queries/zarrQueries.ts +++ b/frontend/src/queries/zarrQueries.ts @@ -196,7 +196,10 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: effectiveVersion + zarrVersion: effectiveVersion, + // This zarr.json is the array metadata, so the codec pipeline is + // already in hand - no second fetch needed. + codecs: (attrs as ZarrV3ArrayMetadata).codecs }, omeZarrUrl: null, availableZarrVersions, @@ -367,6 +370,21 @@ async function fetchZarrMetadata({ log.info('Getting Zarr array for', imageUrl, 'with Zarr version', 2); const arr = await getZarrArray(imageUrl, 2); const shapes = [arr.shape]; + + // Read the compressor so chunk-size warnings know whether the logical + // size is also the stored size. Left undefined on failure, which callers + // treat as compressed. + let compressor: ZarrV2ArrayMetadata['compressor']; + try { + const arrayMeta = (await fetchFileAsJson( + fspName, + zarrayFile.path + )) as ZarrV2ArrayMetadata; + compressor = arrayMeta.compressor; + } catch (error) { + log.trace('Could not fetch .zarray for compressor:', error); + } + return { metadata: { arr, @@ -375,7 +393,8 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: 2 + zarrVersion: 2, + compressor }, omeZarrUrl: null, availableZarrVersions, From 5cee479e56c2641762055f86a2592a7c84bcaff3 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Fri, 21 Aug 2026 13:28:02 -0400 Subject: [PATCH 2/2] fix: address review of the Zarr layout warnings Raise the chunk thresholds to 32 MB uncompressed and 64 MB logical, and cite the published guidance they sit above (image.sc, AWS byte-range, webknossos) rather than one browser's cache default. Drop the browser-cache mechanism from the copy. Chunk size affects viewing because a viewer fetches a whole chunk to show any part of it, which is defensible; the re-download-on-every-access claim was not. Say "inner chunks" when the array is sharded. zarrita resolves the sharding codec when it opens the array, so arr.chunks was already the inner chunk shape and never the shard - only the wording was ambiguous. Remove the axis-order warning. The pinned ome-zarr.js selects the plane by axis name, so a non-spec order renders transposed rather than as a cross-section, and the check fired on channel-last layouts that render correctly. The stated harm was wrong and the remedy - rewriting the array - was out of proportion to a mirrored thumbnail. Co-Authored-By: Claude Opus 5 --- .../unitTests/datasetWarnings.test.ts | 118 ++++++------------ .../components/ui/BrowsePage/MetadataHint.tsx | 18 ++- frontend/src/omezarr-helper.ts | 101 ++++++--------- 3 files changed, 86 insertions(+), 151 deletions(-) diff --git a/frontend/src/__tests__/unitTests/datasetWarnings.test.ts b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts index a86a25d8..e01e7686 100644 --- a/frontend/src/__tests__/unitTests/datasetWarnings.test.ts +++ b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts @@ -15,12 +15,6 @@ const createMetadata = ( ...extra }) as unknown as Metadata; -const axes = (...names: string[]): Partial => ({ - multiscales: [ - { axes: names.map(name => ({ name, type: 'space' })), datasets: [{}, {}] } - ] as unknown as Metadata['multiscales'] -}); - const levels = (count: number): Partial => ({ multiscales: [ { datasets: Array.from({ length: count }, () => ({})) } @@ -45,32 +39,37 @@ describe('getDatasetWarnings: chunk size', () => { expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); }); - it('does not warn about a compressed 16 MB chunk', () => { - // raw/s2: 16 MB inner chunks that zstd takes to ~12 MB on disk. + it('does not warn about a compressed 48 MB chunk', () => { + // 48 MB inner chunks that zstd takes to well under the 32 MB guidance. expect( getDatasetWarnings( - createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + createMetadata([24, 128, 128, 128], 'uint8', SHARDED_ZSTD) ) ).toEqual([]); }); it('holds an uncompressed array to the stricter limit', () => { - // The same 16 MB chunks, but stored raw, so 16 MB is what transfers. + // The same 48 MB chunks, but stored raw, so 48 MB is what transfers. for (const raw of [UNCOMPRESSED_V3, { compressor: null }]) { expect( - getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8', raw)) + getDatasetWarnings(createMetadata([24, 128, 128, 128], 'uint8', raw)) ).toEqual([ - { case: 'zarr-large-chunks', size: '16 MB', compressed: false } + { + case: 'zarr-large-chunks', + size: '48 MB', + compressed: false, + sharded: false + } ]); } }); it('finds a compressor nested inside a sharding codec', () => { // sharding_indexed is structural, so a flat scan would call this - // uncompressed and warn at 16 MB. + // uncompressed and warn at 48 MB. expect( getDatasetWarnings( - createMetadata([8, 128, 128, 128], 'uint8', SHARDED_ZSTD) + createMetadata([24, 128, 128, 128], 'uint8', SHARDED_ZSTD) ) ).toEqual([]); }); @@ -78,7 +77,7 @@ describe('getDatasetWarnings: chunk size', () => { it('assumes compressed when codec metadata was never fetched', () => { // Unknown lands on the permissive limit: a missed warning beats a false one. expect( - getDatasetWarnings(createMetadata([8, 128, 128, 128], 'uint8')) + getDatasetWarnings(createMetadata([24, 128, 128, 128], 'uint8')) ).toEqual([]); }); @@ -87,7 +86,29 @@ describe('getDatasetWarnings: chunk size', () => { expect( getDatasetWarnings(createMetadata([256, 256, 256, 8], 'uint8')) ).toEqual([ - { case: 'zarr-large-chunks', size: '128 MB', compressed: true } + { + case: 'zarr-large-chunks', + size: '128 MB', + compressed: true, + sharded: false + } + ]); + }); + + it('calls out that a sharded array is measured by its inner chunks', () => { + // zarrita resolves the sharding codec, so arr.chunks is the inner chunk + // shape - the shard around it is never what we size. + expect( + getDatasetWarnings( + createMetadata([256, 256, 256, 8], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([ + { + case: 'zarr-large-chunks', + size: '128 MB', + compressed: true, + sharded: true + } ]); }); @@ -130,68 +151,3 @@ describe('getDatasetWarnings: resolution levels', () => { ).toEqual([]); }); }); - -describe('getDatasetWarnings: axis order', () => { - it('accepts spec order', () => { - for (const names of [ - ['t', 'c', 'z', 'y', 'x'], - ['c', 'z', 'y', 'x'], - ['z', 'y', 'x'], - ['y', 'x'] - ]) { - expect( - getDatasetWarnings( - createMetadata([64, 64, 64], 'uint16', axes(...names)) - ) - ).toEqual([]); - } - }); - - it('warns about the seed151 c,x,y,z order', () => { - expect( - getDatasetWarnings( - createMetadata([64, 64, 64, 8], 'uint8', axes('c', 'x', 'y', 'z')) - ) - ).toEqual([ - { - case: 'zarr-axis-order', - axisOrder: 'C, X, Y, Z', - expectedOrder: 'C, Z, Y, X' - } - ]); - }); - - it('warns when the channel axis trails the spatial axes', () => { - expect( - getDatasetWarnings( - createMetadata([64, 64, 64, 8], 'uint8', axes('x', 'y', 'z', 'c')) - ) - ).toEqual([ - { - case: 'zarr-axis-order', - axisOrder: 'X, Y, Z, C', - expectedOrder: 'C, Z, Y, X' - } - ]); - }); - - it('is case insensitive', () => { - expect( - getDatasetWarnings( - createMetadata([64, 64, 64], 'uint16', axes('Z', 'Y', 'X')) - ) - ).toEqual([]); - }); - - it('stays quiet about custom axes it cannot judge', () => { - expect( - getDatasetWarnings( - createMetadata([64, 64, 64], 'uint16', axes('c', 'angle', 'y', 'x')) - ) - ).toEqual([]); - }); - - it('stays quiet about a plain array with no axes', () => { - expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); - }); -}); diff --git a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx index 0650026f..888061c0 100644 --- a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx +++ b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx @@ -14,8 +14,12 @@ type MetadataHintVariant = | { case: 'zarr-query-error'; errorMessage?: string } // Zarr - metadata is valid, but the layout will make viewing awkward | { case: 'zarr-single-level'; size: string } - | { case: 'zarr-large-chunks'; size: string; compressed: boolean } - | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string } + | { + case: 'zarr-large-chunks'; + size: string; + compressed: boolean; + sharded: boolean; + } // N5 - query never fired | { case: 'n5-has-s0-no-attrs' } | { case: 'n5-has-attrs-no-s0' } @@ -80,19 +84,13 @@ function getHintConfig(variant: MetadataHintVariant): HintConfig { return { kind: 'warning', title: 'Only one resolution level', - description: `This dataset declares multiscales but only supplies a single level, for ${variant.size} of data. Viewers must read full-resolution at every zoom level, so viewing the whole image is far more expensive than it needs to be. Generating a multiscale pyramid fixes this.` - }; - case 'zarr-axis-order': - return { - kind: 'warning', - title: 'Axes are not in the order OME-Zarr specifies', - description: `The axes are ordered ${variant.axisOrder}, but the spec requires T, C, Z, Y, X order - here that would be ${variant.expectedOrder}. Many tools take the last two axes to be the image plane, so they will show a cross-section rather than the expected view. Rewriting the dataset with the axes in spec order avoids this.` + description: `This dataset declares multiscales but only supplies a single level, for ${variant.size} of data. Without multiple levels, viewers read the full-resolution data at every zoom level, making viewing slow. Generating a multiscale pyramid fixes this.` }; case 'zarr-large-chunks': return { kind: 'warning', title: 'Chunks may be too large for efficient viewing', - description: `This dataset uses ${variant.size} chunks ${variant.compressed ? '(before compression)' : '(without compression)'}. Chunk files larger than the browser's cache limit are re-downloaded on every access, which makes viewing slow. A final chunk size of 1-10 MB works best.` + description: `This dataset uses ${variant.size} ${variant.sharded ? 'inner chunks' : 'chunks'} ${variant.compressed ? '(before compression)' : '(without compression)'}. Very large chunks make viewing slow, because a viewer must fetch a whole chunk to show any part of it. A stored chunk size of 1-32 MB works best.` }; case 'n5-has-s0-no-attrs': logger.info( diff --git a/frontend/src/omezarr-helper.ts b/frontend/src/omezarr-helper.ts index a135c310..09c43926 100644 --- a/frontend/src/omezarr-helper.ts +++ b/frontend/src/omezarr-helper.ts @@ -30,29 +30,42 @@ export type Metadata = OmeZarrMetadata & { */ export type DatasetWarning = | { case: 'zarr-single-level'; size: string } - | { case: 'zarr-large-chunks'; size: string; compressed: boolean } - | { case: 'zarr-axis-order'; axisOrder: string; expectedOrder: string }; + | { + case: 'zarr-large-chunks'; + size: string; + compressed: boolean; + sharded: boolean; + }; /** - * Chunks above this defeat the browser cache. Applies when the array is stored + * Chunks above this are large enough to slow viewing down: a viewer has to + * fetch a whole chunk to show any part of it. Applies when the array is stored * without compression, so the size we compute is the size that transfers. + * + * Published guidance puts the useful range well below this - the image.sc + * discussion of OME-Zarr chunk sizes lands on 1-10 MB [1], AWS gives 8-16 MB as + * typical for S3 byte-range reads [2], and webknossos recommends 32^3 to 128^3 + * voxel inner chunks [3]. 32 MB is the top of what anyone recommends, so a chunk + * past it is outside the range rather than merely on the large side of it. + * + * [1] https://forum.image.sc/t/should-compression-play-a-role-in-selecting-chunk-sizes-for-ome-zarr-v0-4-datasets/117877 + * [2] https://d1.awsstatic.com/whitepapers/AmazonS3BestPractices.pdf + * [3] https://docs.webknossos.org/webknossos/data/zarr.html */ -export const MAX_CHUNK_BYTES = 10 * 1024 ** 2; +export const MAX_CHUNK_BYTES = 32 * 1024 ** 2; /** * The same limit for compressed arrays, where all we can compute is the logical * (uncompressed) extent and the real transfer is some unknowable fraction of it. - * Set high enough that a typical ratio cannot push a healthy dataset over. + * Doubled, so a chunk has to be outside the recommended range even at a + * conservative 2x ratio before we say anything. */ -export const MAX_LOGICAL_CHUNK_BYTES = 32 * 1024 ** 2; +export const MAX_LOGICAL_CHUNK_BYTES = 64 * 1024 ** 2; /** * Below this, a single-level image is small enough that the missing pyramid * costs nothing worth mentioning. */ export const MAX_SINGLE_LEVEL_BYTES = 1024 ** 3; -/** The axis order OME-Zarr requires, minus any axes the dataset omits. */ -const CANONICAL_AXIS_ORDER = ['t', 'c', 'z', 'y', 'x']; - /** * Whether the array's chunks are compressed on disk. * @@ -73,6 +86,18 @@ function hasCompressionCodec(codecs: NonNullable): boolean { }); } +/** + * Whether the array is sharded. Worth knowing because zarrita resolves the + * sharding codec when it opens the array - `arr.chunks` is then the inner chunk + * shape, the unit a viewer actually fetches, and not the shard around it. The + * warning says so, since "chunk" alone is ambiguous once sharding is in play. + */ +function isSharded(metadata: Metadata): boolean { + return ( + metadata.codecs?.some(codec => codec.name === 'sharding_indexed') ?? false + ); +} + function isStoredCompressed(metadata: Metadata): boolean { if (metadata.codecs) { return hasCompressionCodec(metadata.codecs); @@ -97,37 +122,6 @@ function product(dims: number[]): number { return dims.reduce((total, dim) => total * dim, 1); } -/** - * The axis names, lowercased, or null if any of them is not one of the five the - * spec defines. A dataset using custom axes is not something we can judge. - */ -function getCanonicalAxisNames(metadata: Metadata): string[] | null { - const axes = metadata.multiscales?.[0]?.axes ?? metadata.axes; - if (!axes?.length) { - return null; - } - const names = axes.map(axis => axis.name?.toLowerCase()); - return names.every(name => name && CANONICAL_AXIS_ORDER.includes(name)) - ? (names as string[]) - : null; -} - -/** Whether `names` appears in CANONICAL_AXIS_ORDER order, skipping absent axes. */ -function isCanonicallyOrdered(names: string[]): boolean { - let from = 0; - return names.every(name => { - const index = CANONICAL_AXIS_ORDER.indexOf(name, from); - if (index === -1) { - return false; - } - from = index + 1; - return true; - }); -} - -const formatAxes = (names: string[]) => - names.map(name => name.toUpperCase()).join(', '); - /** * Flag layout choices that make a dataset awkward to view. * @@ -135,14 +129,11 @@ const formatAxes = (names: string[]) => * every zoom level reads full-resolution chunks - the root cause of the incident * that prompted these checks. * - * Chunks past the browser's cache entry limit are re-fetched on every access. - * Sizes are computed from the shape, so they are logical: exact for an - * uncompressed array and an upper bound for a compressed one, which is why the - * limit depends on whether a compressor is in play. - * - * Axis order matters because plenty of tools take the last two axes to be the - * image plane. OME-Zarr requires t, c, z, y, x order for exactly that reason, - * and a dataset that ignores it renders as a cross-section elsewhere. + * Chunks far above the recommended range slow viewing down, because a viewer has + * to fetch a whole chunk to show any part of it. Sizes are computed from the + * shape, so they are logical: exact for an uncompressed array and an upper bound + * for a compressed one, which is why the limit depends on whether a compressor + * is in play. */ export function getDatasetWarnings(metadata: Metadata): DatasetWarning[] { const { arr } = metadata; @@ -173,18 +164,8 @@ export function getDatasetWarnings(metadata: Metadata): DatasetWarning[] { warnings.push({ case: 'zarr-large-chunks', size: formatFileSize(chunkBytes), - compressed - }); - } - - const axisNames = getCanonicalAxisNames(metadata); - if (axisNames && !isCanonicallyOrdered(axisNames)) { - warnings.push({ - case: 'zarr-axis-order', - axisOrder: formatAxes(axisNames), - expectedOrder: formatAxes( - CANONICAL_AXIS_ORDER.filter(name => axisNames.includes(name)) - ) + compressed, + sharded: isSharded(metadata) }); }