Multicam Fixes

.env.default

@@ -46,6 +46,11 @@ COMPOSE_PROFILES=gpu
 #RABBITMQ_MANAGEMENT_URL="http://rabbit:15672/"
 #RABBITMQ_MANAGEMENT_BROKER_URL_TEMPLATE="amqp://{}:{}@rabbit/default"
 
+# Longest time Traefik waits to receive one whole request, body included.
+# Traefik's own default is 60s, which cuts off any upload chunk (64MB by
+# default) that a slow link cannot deliver within a minute.
+#TRAEFIK_READ_TIMEOUT=600s
+
 # Production data bind paths
 #
 #DIVE_PUBLIC_DATA=/var/local/public

client/dive-common/alignedTimeline.spec.ts

@@ -1,6 +1,7 @@
 import type { FrameImage } from './apispec';
 import {
-  buildAlignedTimeline, buildInverseAlignedIndex, canAlign, computeGapGradient, computeGapSlots,
+  buildAlignedTimeline, buildInverseAlignedIndex, buildOffsetTimeline, canAlign,
+  computeGapGradient, computeGapSlots,
 } from './alignedTimeline';
 
 function frame(timestamp?: number): FrameImage {
@@ -209,3 +210,64 @@ describe('alignedTimeline', () => {
     });
   });
 });
+
+/**
+ * Fixed-rig start offsets: the case timestamps can't cover, because video
+ * frames carry none. EO/IR pairs from the same fixed rig are recorded by
+ * independent encoders, so one can start a fraction of a second after the
+ * other; that lag is constant for the whole recording.
+ */
+describe('buildOffsetTimeline', () => {
+  it('pairs a later-starting camera with the reference instant', () => {
+    // B starts 2 frames later: B's frame 2 is the same instant as A's 0.
+    const result = buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 2 });
+    if (!result.aligned) throw new Error('expected aligned');
+    // Slot 0 is the earliest instant ANY camera saw: B's frame 0, before A began.
+    expect(result.slots[0]).toEqual({ A: undefined, B: 0 });
+    expect(result.slots[2]).toEqual({ A: 0, B: 2 });
+    expect(result.slots[4]).toEqual({ A: 2, B: 4 });
+  });
+
+  it('keeps the union, blanking each camera outside its own coverage', () => {
+    const result = buildOffsetTimeline({ A: 3, B: 3 }, { A: 0, B: 2 });
+    if (!result.aligned) throw new Error('expected aligned');
+    // 2 leading slots before A starts + 3 shared + 0 trailing.
+    expect(result.slots).toHaveLength(5);
+    expect(computeGapSlots(result.slots)).toEqual([0, 1, 3, 4]);
+    // The overlap in the middle has both cameras.
+    expect(result.slots[2]).toEqual({ A: 0, B: 2 });
+  });
+
+  it('round-trips through the inverse index the resolver uses', () => {
+    const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: 1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    const inverse = buildInverseAlignedIndex(result.slots);
+    // Whatever slot holds A's frame 2 must hold B's frame 3 -- the same instant.
+    const slotForA2 = inverse.A.get(2) as number;
+    expect(result.slots[slotForA2].B).toBe(3);
+    expect(inverse.B.get(3)).toBe(slotForA2);
+  });
+
+  it('handles a negative offset (reference is the later camera)', () => {
+    const result = buildOffsetTimeline({ A: 4, B: 4 }, { A: 0, B: -1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    expect(result.slots[1]).toEqual({ A: 1, B: 0 });
+  });
+
+  it('declines when nothing needs correcting or there is no pair', () => {
+    // All-zero offsets: the positional path already does this, more cheaply.
+    expect(buildOffsetTimeline({ A: 5, B: 5 }, { A: 0, B: 0 })).toEqual({ aligned: false });
+    // A camera with no frames loaded can't be aligned against.
+    expect(buildOffsetTimeline({ A: 5, B: 0 }, { A: 0, B: 2 })).toEqual({ aligned: false });
+    expect(buildOffsetTimeline({ A: 5 }, { A: 3 })).toEqual({ aligned: false });
+  });
+
+  it('treats a missing camera entry as no offset', () => {
+    // A is absent from the offsets map, so it behaves as A: 0 -- identical to
+    // { A: 0, B: 1 }: one leading slot for B's frame 0, then the pairs.
+    const result = buildOffsetTimeline({ A: 3, B: 3 }, { B: 1 });
+    if (!result.aligned) throw new Error('expected aligned');
+    expect(result.slots[0]).toEqual({ A: undefined, B: 0 });
+    expect(result.slots[1]).toEqual({ A: 0, B: 1 });
+  });
+});

client/dive-common/alignedTimeline.ts

@@ -210,3 +210,71 @@ export function computeGapSlots(slots: AlignedSlot[]): number[] {
   });
   return gaps;
 }
+
+/**
+ * A camera's constant start offset, in its own frames: local frame
+ * `slot + offset` shows the same instant as the reference camera's frame
+ * `slot`. A positive offset means this camera starts LATER -- its frame 0
+ * happens before the reference's frame 0, so it must be read further in.
+ */
+export type CameraFrameOffsets = Record<string, number>;
+
+/**
+ * Build a timeline from fixed per-camera start offsets rather than per-frame
+ * timestamps.
+ *
+ * buildAlignedTimeline needs a timestamp on every frame, which only image
+ * sequences carry (parsed from filenames) -- video panes never qualify, so
+ * they scrub in raw-index lockstep and any recording start offset between
+ * two cameras is baked into the review. On a fixed rig that offset is a
+ * single constant, so one number per camera is enough to line them up, and
+ * emitting it as slots means everything downstream (pane seek, gap
+ * indication, cross-camera frame translation) behaves exactly as it does
+ * for a timestamp-aligned dataset.
+ *
+ * Slots span the UNION of the cameras' coverage: where one camera has run
+ * out (or has not started), its entry is undefined, which the existing gap
+ * handling already renders and blanks correctly, rather than silently
+ * trimming footage off the ends.
+ *
+ * Returns { aligned: false } when fewer than two cameras have frames, or
+ * when every offset is zero -- there is nothing to correct then, so the
+ * caller should stay on the cheaper positional path.
+ */
+export function buildOffsetTimeline(
+  cameraFrameCounts: Record<string, number>,
+  offsets: CameraFrameOffsets,
+): TimelineResult {
+  const cameras = Object.keys(cameraFrameCounts)
+    .filter((camera) => cameraFrameCounts[camera] > 0);
+  if (cameras.length < 2) {
+    return { aligned: false };
+  }
+  if (cameras.every((camera) => (offsets[camera] ?? 0) === 0)) {
+    return { aligned: false };
+  }
+  // Slot s shows camera c's local frame s + offset[c]; that frame exists for
+  // s in [-offset[c], count[c] - offset[c]). Take the union across cameras,
+  // then rebase so the emitted slot array is 0-based.
+  const starts = cameras.map((camera) => -(offsets[camera] ?? 0));
+  const ends = cameras.map(
+    (camera) => cameraFrameCounts[camera] - (offsets[camera] ?? 0),
+  );
+  const base = Math.min(...starts);
+  const total = Math.max(...ends) - base;
+  if (total <= 0) {
+    return { aligned: false };
+  }
+  const slots: AlignedSlot[] = new Array(total);
+  for (let index = 0; index < total; index += 1) {
+    const slot: AlignedSlot = {};
+    cameras.forEach((camera) => {
+      const local = index + base + (offsets[camera] ?? 0);
+      slot[camera] = local >= 0 && local < cameraFrameCounts[camera]
+        ? local
+        : undefined;
+    });
+    slots[index] = slot;
+  }
+  return { aligned: true, slots };
+}

client/dive-common/apispec.ts

@@ -199,6 +199,19 @@ interface SaveDetectionsArgs {
   set?: string;
 }
 
+/** Outcome of shifting one camera's stored annotations onto its time offset. */
+interface CameraFrameOffsetResult {
+  camera: string;
+  /** The camera's start offset in its own frames, now both stored and applied. */
+  offset: number;
+  /** Frames the annotations actually moved: the offset minus what was already applied. */
+  delta: number;
+  tracks: number;
+  groups: number;
+  /** Annotations that had nothing left before frame 0 and were deleted. */
+  dropped: number;
+}
+
 interface SaveAttributeArgs {
   delete: string[];
   upsert: Attribute[];
@@ -332,9 +345,13 @@ interface DatasetConfigMutable {
    * role are absent.
    */
   cameraRoles?: Record<string, CameraRole>;
+  /** Per-camera start offset in its own frames, for recorders that started at different times. */
+  cameraFrameOffsets?: Record<string, number>;
+  /** The part of cameraFrameOffsets already applied to each camera's annotations. */
+  cameraFrameOffsetsApplied?: Record<string, number>;
   error?: string;
 }
-const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'typeHierarchy', 'taxonomySources', 'cameraRoles'];
+const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'cameraFrameOffsets', 'cameraFrameOffsetsApplied', 'typeHierarchy', 'taxonomySources', 'cameraRoles'];
 /**
  * Cross-dataset color/style overrides, reused across every dataset when the
  * "shared" color scope is enabled (see clientSettings.typeSettings.colorScope).
@@ -517,6 +534,13 @@ interface Api {
 
   saveDetections(datasetId: string, args: SaveDetectionsArgs): Promise<unknown>;
   saveConfig(datasetId: string, config: DatasetConfigMutable): Promise<unknown>;
+  /**
+   * Shift one camera's stored annotations onto its time offset, in persistence.
+   * Only the part not yet applied moves; the caller reloads the camera afterwards.
+   */
+  applyCameraFrameOffset(
+    datasetId: string, camera: string, offset: number,
+  ): Promise<CameraFrameOffsetResult>;
   saveAttributes(datasetId: string, args: SaveAttributeArgs): Promise<unknown>;
   saveAttributeTrackFilters(datasetId: string,
     args: SaveAttributeTrackFilterArgs): Promise<unknown>;
@@ -931,6 +955,7 @@ export type {
   PipeMetadata,
   Pipelines,
   SaveDetectionsArgs,
+  CameraFrameOffsetResult,
   SaveAttributeArgs,
   SaveAttributeTrackFilterArgs,
   TrainingConfig,

client/dive-common/autoRegisterSelection.spec.ts

@@ -3,86 +3,70 @@ import { describe, expect, it } from 'vitest';
 import proposeRegistrationFrames from './autoRegisterSelection';
 
 describe('proposeRegistrationFrames', () => {
-  it('spreads candidates across every temporal bin (no single-scene bias)', () => {
-    const frames = proposeRegistrationFrames({
-      counts: [1200, 1200],
-      bins: 12,
-      perBin: 2,
-    });
-    expect(frames.length).toBe(24);
-    // Every 100-frame bin contributes exactly its share: a scene-rich
-    // stretch can never supply all the candidates.
-    for (let bin = 0; bin < 12; bin += 1) {
-      const inBin = frames.filter((f) => f >= bin * 100 && f < (bin + 1) * 100);
-      expect(inBin.length).toBe(2);
-    }
-    expect(frames).toEqual([...frames].sort((a, b) => a - b));
+  it('spreads frames evenly across the whole dataset', () => {
+    const frames = proposeRegistrationFrames({ counts: [1200, 1200], count: 12 });
+    expect(frames).toEqual([50, 150, 250, 350, 450, 550, 650, 750, 850, 950, 1050, 1150]);
   });
 
   it('spans only the shortest camera', () => {
-    const frames = proposeRegistrationFrames({
-      counts: [1000, 300],
-      bins: 10,
-      perBin: 1,
-    });
+    const frames = proposeRegistrationFrames({ counts: [1000, 300], count: 10 });
     expect(Math.max(...frames)).toBeLessThan(300);
     expect(frames.length).toBe(10);
   });
 
-  it('ranks within a bin by inter-camera timestamp skew', () => {
-    // Two cameras, 10 frames, one bin: frame 6 is perfectly synced, frame 3
-    // is close, everything else is badly skewed.
+  it('spreads rather than clustering when every frame is perfectly synced', () => {
+    // Exact timestamps give every frame zero skew; the proposal must still
+    // spread across the dataset, not collapse onto the first frames.
     const base = 1_700_000_000;
-    const camA = Array.from({ length: 10 }, (_, i) => base + i);
-    const camB = camA.map((t, i) => {
-      if (i === 6) return t;
-      if (i === 3) return t + 0.1;
-      return t + 5;
+    const stamps = Array.from({ length: 1200 }, (_, i) => base + i);
+    const frames = proposeRegistrationFrames({
+      counts: [1200, 1200],
+      timestamps: [stamps, [...stamps]],
+      count: 12,
     });
+    expect(frames).toEqual([50, 150, 250, 350, 450, 550, 650, 750, 850, 950, 1050, 1150]);
+  });
+
+  it('excludes frames whose skew exceeds the threshold and spreads over the rest', () => {
+    // Frames 0-4 are hopelessly out of sync; 5-9 are synced.
+    const base = 1_700_000_000;
+    const camA = Array.from({ length: 10 }, (_, i) => base + i);
+    const camB = camA.map((t, i) => (i < 5 ? t + 5 : t));
     const frames = proposeRegistrationFrames({
       counts: [10, 10],
       timestamps: [camA, camB],
-      bins: 1,
-      perBin: 2,
+      count: 5,
       maxSkewSeconds: 0.5,
     });
-    expect(frames).toEqual([3, 6]);
+    expect(frames).toEqual([5, 6, 7, 8, 9]);
   });
 
-  it('drops candidates whose skew exceeds the threshold entirely', () => {
+  it('returns nothing when every frame is out of sync', () => {
     const base = 1_700_000_000;
     const camA = Array.from({ length: 4 }, (_, i) => base + i);
-    const camB = camA.map((t) => t + 10); // hopeless sync everywhere
-    const frames = proposeRegistrationFrames({
-      counts: [4, 4],
-      timestamps: [camA, camB],
-      bins: 1,
-      perBin: 2,
-      maxSkewSeconds: 0.5,
-    });
-    expect(frames).toEqual([]);
+    const camB = camA.map((t) => t + 10);
+    expect(proposeRegistrationFrames({
+      counts: [4, 4], timestamps: [camA, camB], count: 2, maxSkewSeconds: 0.5,
+    })).toEqual([]);
   });
 
-  it('falls back to even spread for frames without timestamps', () => {
+  it('keeps frames whose skew is unknowable', () => {
     const frames = proposeRegistrationFrames({
       counts: [100, 100],
       timestamps: [
         Array.from({ length: 100 }, () => undefined),
         Array.from({ length: 100 }, () => undefined),
       ],
-      bins: 2,
-      perBin: 2,
+      count: 4,
     });
-    expect(frames.length).toBe(4);
-    expect(frames.filter((f) => f < 50).length).toBe(2);
-    expect(frames.filter((f) => f >= 50).length).toBe(2);
+    expect(frames).toEqual([12, 37, 62, 87]);
   });
 
   it('handles degenerate inputs', () => {
-    expect(proposeRegistrationFrames({ counts: [0, 10], bins: 5, perBin: 2 })).toEqual([]);
-    expect(proposeRegistrationFrames({ counts: [], bins: 5, perBin: 2 })).toEqual([]);
-    // More bins than frames: every frame proposed once.
-    const tiny = proposeRegistrationFrames({ counts: [3, 3], bins: 12, perBin: 2 });
-    expect(tiny).toEqual([0, 1, 2]);
+    expect(proposeRegistrationFrames({ counts: [0, 10], count: 5 })).toEqual([]);
+    expect(proposeRegistrationFrames({ counts: [], count: 5 })).toEqual([]);
+    expect(proposeRegistrationFrames({ counts: [10, 10], count: 0 })).toEqual([]);
+    // More frames requested than exist: every frame proposed once.
+    expect(proposeRegistrationFrames({ counts: [3, 3], count: 12 })).toEqual([0, 1, 2]);
   });
 });