diff --git a/README.md b/README.md index a50e968f3..f511dfbc6 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,21 @@ A page will automatically open in your browser and you are ready to go. +## Camera rotation + +If a camera is mounted upside down or sideways, select its **Rotation** in the camera setup on the Calibration page. +Choose 0°, 90°, 180°, or 270° clockwise. For an upside-down wrist camera, choose 180°. + +LeLab saves the rotation for each camera. Setup and teleoperation previews, recorded frames, and inference use the selected rotation. +Profiles without a rotation setting use 0°. + +The resolution setting describes the camera's capture resolution. A 90° or 270° rotation swaps the output dimensions. +For example, a 640 × 480 capture becomes a 480 × 640 observation. The preview shows the complete rotated frame. + +Before inference, select the same rotation used for the training dataset. Check it again if you change the camera binding. +Checkpoint image dimensions do not establish image orientation. For example, 0° and 180° have the same dimensions. +The setting applies to new observations; it does not change existing recordings or checkpoints. + ## Resources - **[LeRobot](https://github.com/huggingface/lerobot):** the underlying library — go here for everything beyond the UI. diff --git a/frontend/src/components/control/BrowserCameraPreview.test.tsx b/frontend/src/components/control/BrowserCameraPreview.test.tsx new file mode 100644 index 000000000..6617378ae --- /dev/null +++ b/frontend/src/components/control/BrowserCameraPreview.test.tsx @@ -0,0 +1,48 @@ +// @vitest-environment jsdom +import React, { act } from "react"; +import { createRoot, Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import BrowserCameraPreview from "./BrowserCameraPreview"; +import CameraRotationSelect from "./CameraRotationSelect"; +vi.mock("@/hooks/useCameraStream", () => ({ useCameraStream: () => ({ videoRef: React.createRef(), hasError: false }) })); +let root: Root; +let container: HTMLDivElement; +beforeEach(() => { Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); container = document.createElement("div"); root = createRoot(container); }); +afterEach(() => act(() => root.unmount())); +test("the Rotation control exposes four clockwise values and emits a number", () => { + const change = vi.fn(); + act(() => root.render()); + const select = container.querySelector("select")!; + expect(container.querySelector("label")!.htmlFor).toBe(select.id); + expect(select.value).toBe("180"); + expect(Array.from(select.options).map((option) => option.text)).toEqual(["0° clockwise", "90° clockwise", "180° clockwise", "270° clockwise"]); + act(() => { select.value = "90"; select.dispatchEvent(new Event("change", { bubbles: true })); }); + expect(change).toHaveBeenCalledWith(90); +}); +test("a quarter turn fits the whole non-square frame and uses actual stream dimensions", () => { + act(() => root.render()); + const video = container.querySelector("video")!, wrapper = video.parentElement!; + expect(wrapper.style.aspectRatio).toBe("480 / 640"); + expect(video.style.objectFit).toBe("contain"); + expect(video.style.maxWidth).toBe("none"); + expect(video.style.transform).toBe("translate(-50%, -50%) rotate(90deg)"); + expect(parseFloat(video.style.width)).toBeCloseTo(640 / 480 * 100); + expect(parseFloat(video.style.height)).toBeCloseTo(480 / 640 * 100); + Object.defineProperty(video, "videoWidth", { value: 1280 }); + Object.defineProperty(video, "videoHeight", { value: 720 }); + act(() => video.dispatchEvent(new Event("loadedmetadata"))); + expect(wrapper.style.aspectRatio).toBe("720 / 1280"); + expect(parseFloat(video.style.width)).toBeCloseTo(1280 / 720 * 100); + expect(parseFloat(video.style.height)).toBeCloseTo(720 / 1280 * 100); +}); +test("device reassignment does not reuse the old device's frame geometry", () => { + act(() => root.render()); + const video = container.querySelector("video")!; + Object.defineProperty(video, "videoWidth", { value: 1920 }); + Object.defineProperty(video, "videoHeight", { value: 1080 }); + act(() => video.dispatchEvent(new Event("loadedmetadata"))); + act(() => root.render()); + expect(video.parentElement!.style.aspectRatio).toBe("640 / 480"); + expect(video.style.width).toBe("100%"); + expect(video.style.transform).toContain("rotate(180deg)"); +}); diff --git a/frontend/src/components/control/BrowserCameraPreview.tsx b/frontend/src/components/control/BrowserCameraPreview.tsx new file mode 100644 index 000000000..0ec1a8a28 --- /dev/null +++ b/frontend/src/components/control/BrowserCameraPreview.tsx @@ -0,0 +1,63 @@ +import React, { useState } from "react"; +import { VideoOff } from "lucide-react"; +import { useCameraStream } from "@/hooks/useCameraStream"; +import { CameraRotation, cameraPreviewGeometry, cameraRotationError } from "@/lib/cameraConfig"; + +interface Props { + deviceId: string; + paused?: boolean; + rotation?: CameraRotation; + width?: number; + height?: number; + className?: string; + fallback?: string; +} + +/** Browser streams contain raw frames, so only browser previews use CSS rotation. */ +export default function BrowserCameraPreview({ + deviceId, + paused = false, + rotation = 0, + width = 640, + height = 480, + className = "", + fallback, +}: Props) { + const rotationError = cameraRotationError({ rotation }); + const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError); + const [metadata, setMetadata] = useState<{ deviceId: string; width: number; height: number } | null>(null); + const dimensions = metadata?.deviceId === deviceId ? metadata : { width, height }; + const geometry = rotationError + ? { aspectRatio: `${width} / ${height}`, videoStyle: {} } + : cameraPreviewGeometry(dimensions.width, dimensions.height, rotation); + const showVideo = !rotationError && !paused && deviceId && !hasError; + return ( +
+ {showVideo ? ( +
+ ); +} diff --git a/frontend/src/components/control/CameraFeed.tsx b/frontend/src/components/control/CameraFeed.tsx index c1d3cbc85..ccf3a333a 100644 --- a/frontend/src/components/control/CameraFeed.tsx +++ b/frontend/src/components/control/CameraFeed.tsx @@ -1,46 +1,22 @@ import React from "react"; -import { VideoOff } from "lucide-react"; -import { useCameraStream } from "@/hooks/useCameraStream"; +import BrowserCameraPreview from "./BrowserCameraPreview"; +import { CameraRotation } from "@/lib/cameraConfig"; interface CameraFeedProps { - /** Browser deviceId to stream. Empty string renders the "no camera" state. */ deviceId: string; - /** Optional caption shown under the feed. */ label?: string; + rotation?: CameraRotation; + width?: number; + height?: number; } -/** Live browser-camera feed bound to a deviceId via getUserMedia. */ -const CameraFeed: React.FC = ({ deviceId, label }) => { - const { videoRef, hasError } = useCameraStream(deviceId, false); - const showVideo = deviceId && !hasError; - - return ( -
-
- {showVideo ? ( -
- {label && ( -
- {label} -
- )} -
- ); -}; +const CameraFeed: React.FC = ({ deviceId, label, rotation, width, height }) => ( +
+ + {label && ( +
{label}
+ )} +
+); export default CameraFeed; diff --git a/frontend/src/components/control/CameraRotationSelect.tsx b/frontend/src/components/control/CameraRotationSelect.tsx new file mode 100644 index 000000000..a664eb862 --- /dev/null +++ b/frontend/src/components/control/CameraRotationSelect.tsx @@ -0,0 +1,30 @@ +import React, { useId } from "react"; +import { Label } from "@/components/ui/label"; +import { CAMERA_ROTATIONS, CameraRotation, cameraRotation } from "@/lib/cameraConfig"; + +interface Props { + rotation: CameraRotation | null; + onChange: (rotation: CameraRotation) => void; + disabled?: boolean; +} + +export default function CameraRotationSelect({ rotation, onChange, disabled }: Props) { + const id = useId(); + return ( +
+ + +
+ ); +} diff --git a/frontend/src/components/control/RecordingCameraFeed.test.tsx b/frontend/src/components/control/RecordingCameraFeed.test.tsx new file mode 100644 index 000000000..a00049334 --- /dev/null +++ b/frontend/src/components/control/RecordingCameraFeed.test.tsx @@ -0,0 +1,27 @@ +// @vitest-environment jsdom +import React, { act } from "react"; +import { createRoot } from "react-dom/client"; +import { expect, test } from "vitest"; +import RecordingCameraFeed from "./RecordingCameraFeed"; +import { recordingCameraWindows } from "@/lib/cameraConfig"; + +test("recording feeds use each camera's output ratio without rotating corrected backend frames", () => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + const windows = recordingCameraWindows(["front", "wrist"], { + front: { type: "opencv", width: 640, height: 480, rotation: 0 }, + wrist: { type: "opencv", width: 640, height: 480, rotation: 90 }, + }, { w: 1000, h: 600 }); + const container = document.createElement("div"), root = createRoot(container); + act(() => root.render(<>{Object.entries(windows).map(([name, dimensions]) => ( + + ))})); + const images = Array.from(container.querySelectorAll("img")); + expect(images).toHaveLength(2); + expect(parseFloat(images[0].parentElement!.style.width) / parseFloat(images[0].parentElement!.style.height)).toBeCloseTo(4 / 3, 2); + expect(parseFloat(images[1].parentElement!.style.width) / parseFloat(images[1].parentElement!.style.height)).toBeCloseTo(3 / 4, 2); + for (const image of images) { + expect(image.className).toContain("object-contain"); + expect(image.style.transform).toBe(""); + } + act(() => root.unmount()); +}); diff --git a/frontend/src/components/control/RecordingCameraFeed.tsx b/frontend/src/components/control/RecordingCameraFeed.tsx new file mode 100644 index 000000000..ec6b3bf8d --- /dev/null +++ b/frontend/src/components/control/RecordingCameraFeed.tsx @@ -0,0 +1,75 @@ +import React, { useState, useEffect, useRef, useCallback } from "react"; + +interface CameraFeedProps { + baseUrl: string; + name: string; + // False during the preparing phase: show an empty slot until the camera is + // connected and streaming. True once recording/resetting, when frames flow. + live: boolean; + // Pixel size computed by the parent to fit the available space. The box is + // sized to the camera's aspect ratio, so the video fills it without letterbox. + width: number; + height: number; +} + +// Renders one recording camera's window at an explicit size. During preparing it +// shows an empty placeholder; once live it plays the backend MJPEG stream. The +// browser renders a `multipart/x-mixed-replace` response natively in an , +// so we just point it at /camera-feed/{name}. If the stream errors before frames +// flow (camera still warming up), retry with a cache-busting key after a delay. +const RecordingCameraFeed: React.FC = ({ + baseUrl, + name, + live, + width, + height, +}) => { + const [reloadKey, setReloadKey] = useState(0); + const [hasError, setHasError] = useState(false); + const retryRef = useRef(null); + + const src = `${baseUrl}/camera-feed/${encodeURIComponent(name)}?k=${reloadKey}`; + + useEffect(() => { + return () => { + if (retryRef.current) window.clearTimeout(retryRef.current); + }; + }, []); + + const handleError = useCallback(() => { + setHasError(true); + if (retryRef.current) window.clearTimeout(retryRef.current); + retryRef.current = window.setTimeout(() => { + setHasError(false); + setReloadKey((k) => k + 1); + }, 1500); + }, []); + + // 0 before the first measurement; skip rendering a zero-size box. + if (width <= 0 || height <= 0) return null; + + return ( +
+ {!live ? ( + Getting ready… + ) : hasError ? ( + Connecting feed… + ) : ( + {`${name} + )} + + {name} + +
+ ); +}; + +export default RecordingCameraFeed; diff --git a/frontend/src/components/control/TeleopCameraPanel.tsx b/frontend/src/components/control/TeleopCameraPanel.tsx index cb48a7144..e386554e4 100644 --- a/frontend/src/components/control/TeleopCameraPanel.tsx +++ b/frontend/src/components/control/TeleopCameraPanel.tsx @@ -33,6 +33,9 @@ const TeleopCameraPanel: React.FC = () => { key: c.id, name: c.name, deviceId: c.device_id, + rotation: c.rotation, + width: c.width, + height: c.height, })); return ( @@ -72,6 +75,9 @@ const TeleopCameraPanel: React.FC = () => { key={`${feed.key}:${reloadKey}`} deviceId={feed.deviceId} label={feed.name} + rotation={feed.rotation} + width={feed.width} + height={feed.height} /> ))} diff --git a/frontend/src/components/landing/CameraBindings.tsx b/frontend/src/components/landing/CameraBindings.tsx index ad6bf6fab..d9b9723aa 100644 --- a/frontend/src/components/landing/CameraBindings.tsx +++ b/frontend/src/components/landing/CameraBindings.tsx @@ -1,5 +1,4 @@ import React from "react"; -import { VideoOff } from "lucide-react"; import { Label } from "@/components/ui/label"; import { Select, @@ -10,38 +9,16 @@ import { } from "@/components/ui/select"; import { AvailableCamera } from "@/hooks/useAvailableCameras"; import { ImageFeatures } from "@/lib/checkpointsApi"; -import { useCameraStream } from "@/hooks/useCameraStream"; - -const CameraThumbnail: React.FC<{ deviceId: string; paused: boolean }> = ({ - deviceId, - paused, -}) => { - const { videoRef, hasError } = useCameraStream(deviceId, paused); - if (paused || hasError || !deviceId) { - return ( -
- - - {paused ? "Released" : "No preview"} - -
- ); - } - return ( -