From a5910d577a2ffec9e3f012eb3729d466a8c4b88e Mon Sep 17 00:00:00 2001 From: Lior Ben Horin Date: Sun, 19 Jul 2026 10:40:49 +0000 Subject: [PATCH 01/11] Add native OpenCV lens-distortion cameras for OVRTX Add OpenCvPinholeDistortionCfg and OpenCvFisheyeDistortionCfg and a distortion field on PinholeCameraCfg so a real camera calibration (fx/fy/cx/cy plus distortion coefficients) can be reproduced in sim. The model is authored at spawn as the omni:lensdistortion USD API, which the RTX/OVRTX renderer honors natively. Make the intrinsics readback distortion-aware so camera.data.intrinsic_matrices reflects a non-square or off-center calibration instead of assuming fx == fy and a centered principal point. The Newton renderer does not yet apply the model and is left as a documented no-op with a warning. --- docs/source/api/lab/isaaclab.sim.spawners.rst | 15 ++ .../opencv-lens-distortion-cameras.minor.rst | 17 ++ .../isaaclab/sensors/camera/camera.py | 47 +++- .../sim/spawners/sensors/__init__.pyi | 12 +- .../isaaclab/sim/spawners/sensors/sensors.py | 65 ++++- .../sim/spawners/sensors/sensors_cfg.py | 130 +++++++++- .../test_camera_opencv_distortion_ovrtx.py | 167 +++++++++++++ .../test/sensors/test_opencv_distortion.py | 229 ++++++++++++++++++ .../opencv-lens-distortion-cameras.rst | 7 + .../renderers/newton_warp_renderer.py | 11 + 10 files changed, 687 insertions(+), 13 deletions(-) create mode 100644 source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst create mode 100644 source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py create mode 100644 source/isaaclab/test/sensors/test_opencv_distortion.py create mode 100644 source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst diff --git a/docs/source/api/lab/isaaclab.sim.spawners.rst b/docs/source/api/lab/isaaclab.sim.spawners.rst index ce82c003e8e5..fd9e5231217f 100644 --- a/docs/source/api/lab/isaaclab.sim.spawners.rst +++ b/docs/source/api/lab/isaaclab.sim.spawners.rst @@ -211,6 +211,9 @@ Sensors PinholeCameraCfg FisheyeCameraCfg + OpenCvDistortionCfg + OpenCvPinholeDistortionCfg + OpenCvFisheyeDistortionCfg .. autofunction:: spawn_camera @@ -222,6 +225,18 @@ Sensors :members: :exclude-members: __init__, func +.. autoclass:: OpenCvDistortionCfg + :members: + :exclude-members: __init__, func + +.. autoclass:: OpenCvPinholeDistortionCfg + :members: + :exclude-members: __init__, func + +.. autoclass:: OpenCvFisheyeDistortionCfg + :members: + :exclude-members: __init__, func + From Files ---------- diff --git a/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst new file mode 100644 index 000000000000..603804a2e5c4 --- /dev/null +++ b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst @@ -0,0 +1,17 @@ +Added +^^^^^ + +* Added :class:`~isaaclab.sim.spawners.sensors.OpenCvPinholeDistortionCfg` and + :class:`~isaaclab.sim.spawners.sensors.OpenCvFisheyeDistortionCfg` and a ``distortion`` field on + :class:`~isaaclab.sim.spawners.sensors.PinholeCameraCfg`, letting a camera carry an OpenCV + ``fx/fy/cx/cy`` + distortion-coefficient calibration. The model is authored on the camera prim as + the ``omni:lensdistortion:*`` USD API, which the RTX/OVRTX renderer honors natively. + +Changed +^^^^^^^ + +* Changed :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices` to read the authored + ``fx/fy/cx/cy`` when an OpenCV lens-distortion model is present, so + :attr:`~isaaclab.sensors.camera.Camera.data` intrinsics reflect a real calibration (non-square + pixels or an off-center principal point) instead of assuming ``fx == fy`` and a centered principal + point. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 58cf4423eaf5..7c36ba0ef7df 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -204,6 +204,8 @@ def __init__(self, cfg: CameraCfg): # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = list() + # Guard so the lens-distortion image-size mismatch warning is emitted at most once. + self._warned_distortion_image_size: bool = False # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # Renderer and render data — assigned in _initialize_impl. @@ -651,18 +653,43 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = for matrix_id, i in enumerate(env_ids_np): # Get corresponding sensor prim sensor_prim = self._sensor_prims[int(i)] - # get camera parameters - # currently rendering does not use aperture offsets or vertical aperture - focal_length = sensor_prim.GetFocalLengthAttr().Get() - horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get() - # get viewport parameters height, width = self.image_shape - # extract intrinsic parameters - f_x = (width * focal_length) / horiz_aperture - f_y = f_x - c_x = width * 0.5 - c_y = height * 0.5 + # Prefer an authored OpenCV lens-distortion model when present: it carries the + # authoritative fx/fy/cx/cy that the RTX/OVRTX renderer projects through. These may be + # non-square (fx != fy) or off-center, unlike the focal-length/aperture readback which + # assumes square pixels and a centered principal point. + raw_prim = sensor_prim.GetPrim() + distortion_model = raw_prim.GetAttribute("omni:lensdistortion:model").Get() + if distortion_model: + prefix = f"omni:lensdistortion:{distortion_model}" + f_x = float(raw_prim.GetAttribute(f"{prefix}:fx").Get()) + f_y = float(raw_prim.GetAttribute(f"{prefix}:fy").Get()) + c_x = float(raw_prim.GetAttribute(f"{prefix}:cx").Get()) + c_y = float(raw_prim.GetAttribute(f"{prefix}:cy").Get()) + # the intrinsics are reported in the calibrated pixel space; warn once if the render + # resolution differs, since the reported matrix will not match the rendered pixels. + image_size = raw_prim.GetAttribute(f"{prefix}:imageSize").Get() + if image_size is not None and (int(image_size[0]), int(image_size[1])) != (width, height): + if not self._warned_distortion_image_size: + logger.warning( + "Camera lens-distortion 'imageSize' %s does not match the render resolution" + " (%d, %d). The reported intrinsic matrix is in the calibrated pixel space.", + (int(image_size[0]), int(image_size[1])), + width, + height, + ) + self._warned_distortion_image_size = True + else: + # get camera parameters + # currently rendering does not use aperture offsets or vertical aperture + focal_length = sensor_prim.GetFocalLengthAttr().Get() + horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get() + # extract intrinsic parameters + f_x = (width * focal_length) / horiz_aperture + f_y = f_x + c_x = width * 0.5 + c_y = height * 0.5 # create intrinsic matrix for depth linear intrinsic_matrices[matrix_id, 0, 0] = f_x intrinsic_matrices[matrix_id, 0, 2] = c_x diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi index ba5b96a44c7d..20c18ff9bfa8 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi @@ -7,9 +7,19 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", ] from .sensors import spawn_camera, spawn_sensor_frame -from .sensors_cfg import FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg +from .sensors_cfg import ( + FisheyeCameraCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + SensorFrameCfg, +) diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py index db68d21d8a90..a1e6ccbf29ac 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py @@ -8,7 +8,7 @@ import logging from typing import TYPE_CHECKING -from pxr import Sdf, Usd +from pxr import Gf, Sdf, Usd from isaaclab.sim.utils import change_prim_property, clone, create_prim, get_current_stage from isaaclab.utils import to_camel_case @@ -48,6 +48,65 @@ """ +# OpenCV lens-distortion models authored as the ``omni:lensdistortion:*`` USD API. The RTX/OVRTX +# renderer honors these attributes natively; they are read back into ``camera.data.intrinsic_matrices`` +# by :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices`. +_OPENCV_DISTORTION_API_SCHEMAS = { + "opencvPinhole": "OmniLensDistortionOpenCvPinholeAPI", + "opencvFisheye": "OmniLensDistortionOpenCvFisheyeAPI", +} +"""Maps an OpenCV distortion model discriminator to its applied USD API schema name.""" + +_OPENCV_DISTORTION_COEFFS = { + "opencvPinhole": ("k1", "k2", "k3", "k4", "k5", "k6", "p1", "p2", "s1", "s2", "s3", "s4"), + "opencvFisheye": ("k1", "k2", "k3", "k4"), +} +"""Maps an OpenCV distortion model discriminator to its distortion-coefficient field names.""" + + +def _author_opencv_distortion(prim: Usd.Prim, cfg: sensors_cfg.OpenCvDistortionCfg) -> None: + """Author an OpenCV lens-distortion model on a camera prim as the ``omni:lensdistortion:*`` API. + + The attributes are authored explicitly (not through the generic camelCase loop of + :func:`spawn_camera`) because their names are namespaced (e.g. ``omni:lensdistortion:opencvPinhole:k1``) + and cannot be produced by :func:`~isaaclab.utils.to_camel_case`. Applying the schema only edits prim + metadata, so it survives ``stage.ExportToString()`` and does not require the schema to be registered. + + Args: + prim: The camera prim to author the distortion model on. + cfg: The OpenCV distortion configuration. + + Raises: + ValueError: If the distortion ``model`` is not a supported OpenCV model. + """ + if cfg.model not in _OPENCV_DISTORTION_API_SCHEMAS: + raise ValueError( + f"Unsupported OpenCV distortion model: '{cfg.model}'. Supported models are:" + f" {list(_OPENCV_DISTORTION_API_SCHEMAS)}." + ) + prefix = f"omni:lensdistortion:{cfg.model}" + + # apply the schema and set the model discriminator token + prim.AddAppliedSchema(_OPENCV_DISTORTION_API_SCHEMAS[cfg.model]) + + def _set_attr(name: str, type_name: Sdf.ValueTypeName, value) -> None: + attr = prim.GetAttribute(name) or prim.CreateAttribute(name, type_name) + attr.Set(value) + + _set_attr("omni:lensdistortion:model", Sdf.ValueTypeNames.Token, cfg.model) + _set_attr( + f"{prefix}:imageSize", + Sdf.ValueTypeNames.Int2, + Gf.Vec2i(int(cfg.image_size[0]), int(cfg.image_size[1])), + ) + for name in ("fx", "fy", "cx", "cy"): + _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, float(getattr(cfg, name))) + # coefficients are muted (authored as zero) unless apply_lens_distortion is set + for name in _OPENCV_DISTORTION_COEFFS[cfg.model]: + value = float(getattr(cfg, name)) if cfg.apply_lens_distortion else 0.0 + _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, value) + + @clone def spawn_camera( prim_path: str, @@ -119,6 +178,7 @@ def spawn_camera( "semantic_tags", "from_intrinsic_matrix", "spawn_path", + "distortion", ] # get camera prim prim = stage.GetPrimAtPath(prim_path) @@ -143,6 +203,9 @@ def spawn_camera( prim_prop_name = to_camel_case(param_name, to="cC") # get attribute from the class prim.GetAttribute(prim_prop_name).Set(param_value) + # author the OpenCV lens-distortion model (renderer-agnostic; RTX/OVRTX honors it natively) + if getattr(cfg, "distortion", None) is not None: + _author_opencv_distortion(prim, cfg.distortion) # return the prim return prim diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py index 6a1741675074..82f0274aedd6 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import MISSING from typing import Literal import isaaclab.utils.sensors as sensor_utils @@ -13,6 +14,120 @@ from isaaclab.utils.configclass import configclass +@configclass +class OpenCvDistortionCfg: + """Base configuration for an OpenCV lens-distortion model carried on a camera cfg. + + The distortion model is renderer-agnostic: it is stored on the camera spawn configuration + and each renderer decides how to consume it. Under the RTX/OVRTX renderer the fields are + authored as the ``omni:lensdistortion:*`` USD API, which the renderer honors natively. The + Newton renderer does not yet apply this model. + + The intrinsic parameters (:attr:`fx`, :attr:`fy`, :attr:`cx`, :attr:`cy`) follow the OpenCV + convention. When a distortion model is present, they take precedence over the focal-length + and aperture projection of :class:`PinholeCameraCfg`. + + This base class is not meant to be used directly. Use one of the concrete models, e.g. + :class:`OpenCvPinholeDistortionCfg` or :class:`OpenCvFisheyeDistortionCfg`. + """ + + model: str = MISSING + """Discriminator selecting the OpenCV distortion model. Set by each concrete sub-config.""" + + fx: float = MISSING + """Focal length along the image x-axis (in pixels).""" + + fy: float = MISSING + """Focal length along the image y-axis (in pixels).""" + + cx: float = MISSING + """Principal point offset along the image x-axis (in pixels).""" + + cy: float = MISSING + """Principal point offset along the image y-axis (in pixels).""" + + image_size: tuple[int, int] = MISSING + """Calibrated image size as ``(width, height)`` (in pixels).""" + + apply_lens_distortion: bool = True + """Whether to apply the distortion coefficients. Defaults to True. + + If False, the distortion coefficients are authored as zero while the intrinsic parameters + (:attr:`fx`, :attr:`fy`, :attr:`cx`, :attr:`cy`) are kept. This produces an undistorted image + that still uses the calibrated OpenCV intrinsics, which is useful for isolating the effect of + the lens distortion. + """ + + +@configclass +class OpenCvPinholeDistortionCfg(OpenCvDistortionCfg): + """OpenCV pinhole lens-distortion model (radial, tangential and thin-prism terms). + + Corresponds to ``OmniLensDistortionOpenCvPinholeAPI`` under the RTX/OVRTX renderer. The full + coefficient set of the OpenCV rational model is exposed; unused coefficients default to zero. + """ + + model: str = "opencvPinhole" + + k1: float = 0.0 + """First radial distortion coefficient. Defaults to 0.0.""" + + k2: float = 0.0 + """Second radial distortion coefficient. Defaults to 0.0.""" + + k3: float = 0.0 + """Third radial distortion coefficient. Defaults to 0.0.""" + + k4: float = 0.0 + """Fourth radial distortion coefficient (rational model). Defaults to 0.0.""" + + k5: float = 0.0 + """Fifth radial distortion coefficient (rational model). Defaults to 0.0.""" + + k6: float = 0.0 + """Sixth radial distortion coefficient (rational model). Defaults to 0.0.""" + + p1: float = 0.0 + """First tangential distortion coefficient. Defaults to 0.0.""" + + p2: float = 0.0 + """Second tangential distortion coefficient. Defaults to 0.0.""" + + s1: float = 0.0 + """First thin-prism distortion coefficient. Defaults to 0.0.""" + + s2: float = 0.0 + """Second thin-prism distortion coefficient. Defaults to 0.0.""" + + s3: float = 0.0 + """Third thin-prism distortion coefficient. Defaults to 0.0.""" + + s4: float = 0.0 + """Fourth thin-prism distortion coefficient. Defaults to 0.0.""" + + +@configclass +class OpenCvFisheyeDistortionCfg(OpenCvDistortionCfg): + """OpenCV fisheye lens-distortion model. + + Corresponds to ``OmniLensDistortionOpenCvFisheyeAPI`` under the RTX/OVRTX renderer. + """ + + model: str = "opencvFisheye" + + k1: float = 0.0 + """First fisheye distortion coefficient. Defaults to 0.0.""" + + k2: float = 0.0 + """Second fisheye distortion coefficient. Defaults to 0.0.""" + + k3: float = 0.0 + """Third fisheye distortion coefficient. Defaults to 0.0.""" + + k4: float = 0.0 + """Fourth fisheye distortion coefficient. Defaults to 0.0.""" + + @configclass class PinholeCameraCfg(SpawnerCfg): """Configuration parameters for a USD camera prim with pinhole camera settings. @@ -30,7 +145,20 @@ class PinholeCameraCfg(SpawnerCfg): """Type of projection to use for the camera. Defaults to "pinhole". Note: - Currently only "pinhole" is supported. + The stock projection is ``"pinhole"``. An OpenCV ``fx/fy/cx/cy`` + distortion-coefficient + intrinsic model can be applied on top via :attr:`distortion` (see + :class:`OpenCvPinholeDistortionCfg` / :class:`OpenCvFisheyeDistortionCfg`), which the + RTX/OVRTX renderer honors natively. + """ + + distortion: OpenCvDistortionCfg | None = None + """OpenCV lens-distortion model to author on the camera. Defaults to None (no distortion). + + When set, the OpenCV intrinsics and distortion coefficients are authored on the camera prim. + Under the RTX/OVRTX renderer they drive the projection natively and, when a real calibration is + used (``fx != fy`` or an off-center principal point), take precedence over the focal-length and + aperture projection. The Newton renderer does not yet apply this model; the camera renders + undistorted there. """ clipping_range: tuple[float, float] = (0.01, 1e6) diff --git a/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py new file mode 100644 index 000000000000..d5f085b27fc8 --- /dev/null +++ b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py @@ -0,0 +1,167 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Validate that the ``ovrtx`` renderer honors the OpenCV lens-distortion camera model natively. + +The grid-textured ground plane is rendered through a camera carrying an OpenCV pinhole calibration. +With the distortion coefficients applied, OVRTX (which reads ``omni:lensdistortion:opencvPinhole:*`` +from the prim) bows the straight grid lines; with the coefficients muted, the same camera renders them +straight. The two renders must therefore differ meaningfully. The reconstructed ``intrinsic_matrices`` +are also checked end-to-end against the authored, non-square, off-center calibration. + +Notes: + * Runs **kit-less**: this test does not call :class:`~isaaclab.app.AppLauncher`. ``ovrtx`` and Isaac + Sim Kit ship conflicting RTX hydra libraries that cannot co-load; see + :func:`isaaclab.app.sim_launcher.launch_simulation`. + * Uses Newton physics because ``ovrtx`` is incompatible with Kit/Isaac Sim. + * The distorted and reference passes each build their own :class:`SimulationContext`; a single + OVRTX render context applies one lens model, so the two passes must not share a process context. +""" + +import importlib.util + +import numpy as np +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.rendering] + +_REQUIRED_MODULES = ("isaaclab_ov", "ovrtx", "isaaclab_newton") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] +_SKIP_MISSING_OVRTX = pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", +) + +if not _MISSING_MODULES: + import torch + from isaaclab_newton.physics.mjwarp_manager_cfg import MJWarpSolverCfg + from isaaclab_newton.physics.newton_manager_cfg import NewtonCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + + import isaaclab.sim as sim_utils + from isaaclab.assets import AssetBaseCfg, RigidObjectCfg + from isaaclab.scene import InteractiveScene, InteractiveSceneCfg + from isaaclab.sensors import Camera, CameraCfg + from isaaclab.sim import SimulationCfg + from isaaclab.sim.spawners.sensors.sensors_cfg import OpenCvPinholeDistortionCfg, PinholeCameraCfg + from isaaclab.utils.configclass import configclass + from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix + +SIM_DT = 1.0 / 60.0 +WIDTH, HEIGHT = 640, 480 +WARMUP_STEPS = 4 + +# Real SO-101 wrist-camera calibration (fx != fy, off-center principal point). +_CALIB = dict(fx=339.26592887, fy=338.82010626, cx=323.55809091, cy=250.27360914) +_COEFFS = dict(k1=0.07702322, k2=-0.13605453, k3=0.05163219, p1=-0.00024938, p2=-0.00175006) +# scale the (mild) real coefficients so the barrel effect is unambiguous in the assertion +_K_SCALE = 15.0 + +_CAM_EYE = (0.0, 0.0, 2.5) +_CAM_TARGET = (1.75, 0.0, 0.0) + + +if not _MISSING_MODULES: + + @configclass + class _DistortionSceneCfg(InteractiveSceneCfg): + """The grid-textured ground plane, a dome light and an off-screen anchor body for Newton.""" + + ground = AssetBaseCfg(prim_path="/World/ground", spawn=sim_utils.GroundPlaneCfg()) + dome_light = AssetBaseCfg( + prim_path="/World/DomeLight", + spawn=sim_utils.DomeLightCfg(intensity=2000.0, color=(0.9, 0.9, 0.9)), + ) + anchor = RigidObjectCfg( + prim_path="{ENV_REGEX_NS}/Anchor", + spawn=sim_utils.CuboidCfg( + size=(0.01, 0.01, 0.01), + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(mass=0.001), + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=sim_utils.RigidBodyMaterialCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.0, 0.0, 0.0)), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=(0.0, 0.0, -100.0)), + ) + + +def _render_grid(apply_lens_distortion: bool, device: str) -> tuple[np.ndarray, np.ndarray]: + """Render the ground-plane grid through an OpenCV-calibrated OVRTX camera; return ``(rgb, K)``.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext( + SimulationCfg(dt=SIM_DT, physics=NewtonCfg(solver_cfg=MJWarpSolverCfg(), num_substeps=1), device=device) + ) + scene = InteractiveScene(_DistortionSceneCfg(num_envs=1, env_spacing=20.0)) + + rot = tuple( + quat_from_matrix( + create_rotation_matrix_from_view(torch.tensor([_CAM_EYE]), torch.tensor([_CAM_TARGET]), up_axis="Z") + )[0].tolist() + ) + distortion = OpenCvPinholeDistortionCfg( + image_size=(WIDTH, HEIGHT), + apply_lens_distortion=apply_lens_distortion, + **_CALIB, + **{name: value * _K_SCALE for name, value in _COEFFS.items()}, + ) + camera = Camera( + CameraCfg( + prim_path="/World/envs/env_.*/Camera", + update_period=0.0, + height=HEIGHT, + width=WIDTH, + data_types=["rgb"], + offset=CameraCfg.OffsetCfg(pos=_CAM_EYE, rot=rot, convention="opengl"), + spawn=PinholeCameraCfg(focal_length=13.6, clipping_range=(0.001, 20.0), distortion=distortion), + renderer_cfg=OVRTXRendererCfg(), + ) + ) + try: + sim.reset() + for _ in range(WARMUP_STEPS): + sim.step() + camera.update(SIM_DT, force_recompute=True) + rgb = camera.data.output["rgb"].torch[0].detach().cpu().float().numpy()[..., :3] + intrinsics = camera.data.intrinsic_matrices.torch[0].detach().cpu().numpy() + return rgb, intrinsics + finally: + del camera + del scene + sim.stop() + sim.clear_instance() + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_distortion_changes_ovrtx_render(device): + """OVRTX must render the distorted and zero-coefficient cameras meaningfully differently.""" + distorted, _ = _render_grid(apply_lens_distortion=True, device=device) + reference, _ = _render_grid(apply_lens_distortion=False, device=device) + + assert distorted.shape == (HEIGHT, WIDTH, 3) + # both frames render content (not degenerate) + assert distorted.std() > 1.0 + assert reference.std() > 1.0 + # OVRTX applied the lens distortion: the two renders differ well beyond render noise + mean_abs_diff = np.abs(distorted.astype(np.float32) - reference.astype(np.float32)).mean() + assert mean_abs_diff > 2.0, f"distorted vs reference differ by only {mean_abs_diff:.3f}/255" + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_distortion_intrinsics_match_authored_ovrtx(device): + """The OVRTX camera reports intrinsics matching the authored, non-square, off-center calibration.""" + _rgb, k = _render_grid(apply_lens_distortion=True, device=device) + + assert k[0, 0] == pytest.approx(_CALIB["fx"], abs=1e-2) + assert k[1, 1] == pytest.approx(_CALIB["fy"], abs=1e-2) + assert k[0, 2] == pytest.approx(_CALIB["cx"], abs=1e-2) + assert k[1, 2] == pytest.approx(_CALIB["cy"], abs=1e-2) + # not the stock fx == fy / centered-principal-point collapse + assert k[0, 0] != k[1, 1] + assert k[0, 2] != pytest.approx(WIDTH / 2) diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py new file mode 100644 index 000000000000..4312ea213c16 --- /dev/null +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -0,0 +1,229 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Kit-less tests for the renderer-agnostic OpenCV lens-distortion camera model. + +Two renderer-independent code paths are covered without a running simulation app, renderer or GPU: + +* Authoring: :func:`isaaclab.sim.spawners.sensors.spawn_camera` with a ``distortion`` cfg authors the + ``omni:lensdistortion:*`` USD API on the camera prim (which the RTX/OVRTX renderer round-trips + through the USD export it loads). +* Readback: :meth:`isaaclab.sensors.camera.Camera._update_intrinsic_matrices` reconstructs + ``camera.data.intrinsic_matrices`` from the authored ``fx/fy/cx/cy`` (which may be non-square or + off-center) instead of assuming ``fx == fy`` and a centered principal point. + +The renderer actually rendering *through* the distortion is a backend-specific concern covered in +``test_camera_opencv_distortion_ovrtx.py``. +""" + +from __future__ import annotations + +import importlib.util +from types import SimpleNamespace + +import numpy as np +import pytest + +_REQUIRED_MODULES = ("isaaclab", "pxr", "warp") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] + +pytestmark = pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", +) + +if not _MISSING_MODULES: + from pxr import Gf, Sdf, Usd, UsdGeom + + import isaaclab.sim as sim_utils + from isaaclab.sensors.camera.camera import Camera + from isaaclab.sim.spawners.sensors.sensors import spawn_camera + from isaaclab.sim.spawners.sensors.sensors_cfg import ( + FisheyeCameraCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + ) + + +# Real SO-101 wrist-camera calibration: exercises fx != fy and an off-center principal point, which +# stock Isaac Lab camera cfgs cannot express. +_PINHOLE_CALIB = dict( + fx=339.26592887, + fy=338.82010626, + cx=323.55809091, + cy=250.27360914, + image_size=(640, 480), + k1=0.07702322, + k2=-0.13605453, + k3=0.05163219, + p1=-0.00024938, + p2=-0.00175006, +) + + +""" +Authoring: spawn_camera with a distortion cfg. +""" + + +def _spawn_camera_on_new_stage(cfg, prim_path="/World/envs/env_0/Camera"): + """Create a fresh in-memory stage and spawn a camera prim carrying ``cfg`` on it.""" + stage = sim_utils.create_new_stage() + UsdGeom.Xform.Define(stage, "/World") + UsdGeom.Xform.Define(stage, "/World/envs") + UsdGeom.Xform.Define(stage, "/World/envs/env_0") + # spawn_camera is @clone-decorated; call the undecorated function to spawn a single prim. + prim = spawn_camera.__wrapped__(prim_path, cfg) + return stage, prim + + +def test_pinhole_distortion_authors_opencv_api(): + """Spawning a pinhole camera with a distortion cfg authors the OpenCV pinhole USD API.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=True, **_PINHOLE_CALIB)) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvPinhole" + assert prim.GetAttribute("omni:lensdistortion:model").Get() == "opencvPinhole" + assert "OmniLensDistortionOpenCvPinholeAPI" in prim.GetMetadata("apiSchemas").GetAppliedItems() + # intrinsics + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(_PINHOLE_CALIB["fx"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:fy").Get() == pytest.approx(_PINHOLE_CALIB["fy"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:cx").Get() == pytest.approx(_PINHOLE_CALIB["cx"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:cy").Get() == pytest.approx(_PINHOLE_CALIB["cy"], abs=1e-3) + assert tuple(prim.GetAttribute(f"{prefix}:imageSize").Get()) == (640, 480) + # distortion coefficients (applied) + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(_PINHOLE_CALIB["k1"], abs=1e-6) + assert prim.GetAttribute(f"{prefix}:p2").Get() == pytest.approx(_PINHOLE_CALIB["p2"], abs=1e-6) + # unused higher-order coefficients default to zero + assert prim.GetAttribute(f"{prefix}:k6").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:s4").Get() == pytest.approx(0.0) + + +def test_apply_lens_distortion_false_zeros_coefficients(): + """With ``apply_lens_distortion=False`` the intrinsics are kept but coefficients are muted to zero.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=False, **_PINHOLE_CALIB)) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvPinhole" + # intrinsics are still authored + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(_PINHOLE_CALIB["fx"], abs=1e-3) + # coefficients are zeroed despite non-zero cfg values + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:k2").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:p1").Get() == pytest.approx(0.0) + + +def test_fisheye_distortion_authors_opencv_fisheye_api(): + """A distortion cfg with the fisheye model authors the OpenCV fisheye USD API.""" + cfg = FisheyeCameraCfg( + distortion=OpenCvFisheyeDistortionCfg( + fx=300.0, fy=300.0, cx=320.0, cy=240.0, image_size=(640, 480), k1=0.1, k2=0.02, k3=0.0, k4=0.0 + ) + ) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvFisheye" + assert prim.GetAttribute("omni:lensdistortion:model").Get() == "opencvFisheye" + assert "OmniLensDistortionOpenCvFisheyeAPI" in prim.GetMetadata("apiSchemas").GetAppliedItems() + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(300.0) + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(0.1) + assert prim.GetAttribute(f"{prefix}:k4").Get() == pytest.approx(0.0) + + +def test_distortion_attributes_survive_usd_export(): + """The authored OpenCV distortion API survives serialization, which the RTX/OVRTX path consumes.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=True, **_PINHOLE_CALIB)) + stage, _prim = _spawn_camera_on_new_stage(cfg) + + exported = stage.ExportToString() + assert "OmniLensDistortionOpenCvPinholeAPI" in exported + assert "omni:lensdistortion:opencvPinhole:fx" in exported + assert "omni:lensdistortion:model" in exported + + +def test_camera_without_distortion_authors_no_opencv_api(): + """A camera cfg without a distortion model does not author any ``omni:lensdistortion`` attributes.""" + cfg = PinholeCameraCfg() + _stage, prim = _spawn_camera_on_new_stage(cfg) + + assert not prim.GetAttribute("omni:lensdistortion:model").IsValid() + applied = prim.GetMetadata("apiSchemas") + applied_items = list(applied.GetAppliedItems()) if applied else [] + assert not any("LensDistortion" in item for item in applied_items) + + +""" +Readback: Camera._update_intrinsic_matrices with an authored distortion model. +""" + + +def _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width=640, height=480): + """Build an in-memory ``UsdGeom.Camera`` carrying an OpenCV pinhole distortion model.""" + stage = Usd.Stage.CreateInMemory() + cam = UsdGeom.Camera.Define(stage, "/Camera") + prim = cam.GetPrim() + prim.CreateAttribute("omni:lensdistortion:model", Sdf.ValueTypeNames.Token).Set("opencvPinhole") + prefix = "omni:lensdistortion:opencvPinhole" + for name, value in (("fx", fx), ("fy", fy), ("cx", cx), ("cy", cy)): + prim.CreateAttribute(f"{prefix}:{name}", Sdf.ValueTypeNames.Float).Set(float(value)) + prim.CreateAttribute(f"{prefix}:imageSize", Sdf.ValueTypeNames.Int2).Set(Gf.Vec2i(width, height)) + return stage, cam + + +def _read_back_intrinsics(cam, width, height): + """Invoke :meth:`Camera._update_intrinsic_matrices` on a minimal fake camera and return K.""" + captured = {} + + def _capture_state(env_ids=None, intrinsics_src=None, update_intrinsics=False): + captured["K"] = intrinsics_src.numpy() + + fake = Camera.__new__(Camera) + fake._sensor_prims = [cam] + fake._device = "cpu" + fake._warned_distortion_image_size = False + fake.cfg = SimpleNamespace(height=height, width=width) + fake._resolve_env_ids_np = lambda env_ids: np.array([0]) + fake._update_camera_state = _capture_state + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + Camera._update_intrinsic_matrices(fake) + return captured["K"][0] + + +def test_readback_uses_authored_fx_fy_cx_cy(): + """The reconstructed intrinsic matrix reflects the authored, non-square, off-center calibration.""" + fx, fy, cx, cy = 339.26592887, 338.82010626, 323.55809091, 250.27360914 + width, height = 640, 480 + _stage, cam = _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width, height) + + k = _read_back_intrinsics(cam, width, height) + + assert k[0, 0] == pytest.approx(fx, abs=1e-2) + assert k[1, 1] == pytest.approx(fy, abs=1e-2) + assert k[0, 2] == pytest.approx(cx, abs=1e-2) + assert k[1, 2] == pytest.approx(cy, abs=1e-2) + assert k[2, 2] == pytest.approx(1.0) + + +def test_readback_preserves_non_square_and_off_center(): + """The readback must not collapse to the stock ``fx == fy`` / centered principal point.""" + fx, fy, cx, cy = 339.26592887, 338.82010626, 323.55809091, 250.27360914 + width, height = 640, 480 + _stage, cam = _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width, height) + + k = _read_back_intrinsics(cam, width, height) + + # non-square pixels are preserved (the old readback forced fx == fy) + assert k[0, 0] != k[1, 1] + # principal point is off-center (the old readback forced width/2, height/2) + assert k[0, 2] != pytest.approx(width / 2) + assert k[1, 2] != pytest.approx(height / 2) diff --git a/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst b/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst new file mode 100644 index 000000000000..c19be2eddb53 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Added a warning when a camera cfg carries an OpenCV lens-distortion model + (``spawn.distortion``) under the Newton renderer, which does not yet apply the model; the camera + renders undistorted. The distortion cfg is renderer-agnostic and remains a documented extension + point for a future Newton implementation. diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 089be5ffc313..2a031359e51e 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -384,6 +384,17 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: owns the sentinel-resolution + cfg-normalization step. Newton has no USD-side overrides to author beyond this. """ + # NOTE: OpenCV lens distortion (``spawn.distortion``) is not yet applied by the Newton + # renderer. The distortion cfg is renderer-agnostic and could be piped through Newton's warp + # ray-tracing utilities here in the future; for now the camera renders undistorted. This is + # the intended extension point. + spawn = getattr(spec.cfg, "spawn", None) + if getattr(spawn, "distortion", None) is not None: + logger.warning( + "OpenCV lens distortion is set on the camera cfg but is not yet applied by the Newton" + " renderer; the camera will render undistorted. Use the RTX/OVRTX renderer to apply" + " the distortion model." + ) if spec.cfg.isp_cfg is None: return try: From c1e2f95097ed44bab3607f59453f85d36d8ab6f2 Mon Sep 17 00:00:00 2001 From: Lior Ben Horin Date: Sun, 19 Jul 2026 10:47:52 +0000 Subject: [PATCH 02/11] Re-export OpenCV distortion cfgs from isaaclab.sim Propagate OpenCvDistortionCfg, OpenCvPinholeDistortionCfg and OpenCvFisheyeDistortionCfg through the isaaclab.sim.spawners and isaaclab.sim lazy-export stubs, so they are importable as isaaclab.sim. like the sibling PinholeCameraCfg and FisheyeCameraCfg. --- source/isaaclab/isaaclab/sim/__init__.pyi | 6 ++++++ source/isaaclab/isaaclab/sim/spawners/__init__.pyi | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/isaaclab/sim/__init__.pyi b/source/isaaclab/isaaclab/sim/__init__.pyi index f145f9879daf..8496bca7f39f 100644 --- a/source/isaaclab/isaaclab/sim/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/__init__.pyi @@ -137,6 +137,9 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", "spawn_capsule", @@ -332,6 +335,9 @@ from .spawners import ( MjcfFileCfg, MultiAssetSpawnerCfg, MultiUsdFileCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, PhysicsMaterialCfg, PinholeCameraCfg, PreviewSurfaceCfg, diff --git a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi index c936d166ba3c..c2d19f0cbde4 100644 --- a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi @@ -54,6 +54,9 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", "spawn_capsule", @@ -126,7 +129,16 @@ from .meshes import ( MeshRectangleCfg, MeshSphereCfg, ) -from .sensors import spawn_camera, spawn_sensor_frame, FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg +from .sensors import ( + spawn_camera, + spawn_sensor_frame, + FisheyeCameraCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + SensorFrameCfg, +) from .shapes import ( spawn_capsule, spawn_cone, From c2499de865ae9085f08d0d2a6ba48558d22a6d37 Mon Sep 17 00:00:00 2001 From: Lior Ben Horin Date: Sun, 19 Jul 2026 10:56:52 +0000 Subject: [PATCH 03/11] Add Lior Ben Horin to CONTRIBUTORS.md --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index da5d955d0f9d..d5c40dd7057a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -125,6 +125,7 @@ Guidelines for modifications: * Krishna Lakhi * Lin He * Lionel Gulich +* Lior Ben Horin * Lorenz Wellhausen * Lotus Li * Louis Le Lay From 56003296ad3bfe276a22d7f1c35b1009288e156d Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 15:23:53 +0300 Subject: [PATCH 04/11] Harden OpenCV lens-distortion intrinsic readback Fall back to the focal-length/aperture projection when a camera prim carries the omni:lensdistortion:model token without the fx/fy/cx/cy intrinsics, instead of dereferencing a None attribute value on every update step and raising TypeError continuously. Key the image-size mismatch warning on the authored size and include the prim path and environment index, so a genuinely different calibration in another environment is surfaced rather than masked by the first prim's one-time flag. Add regression tests covering the missing-intrinsics fallback and the per-size warning behavior. --- .../isaaclab/sensors/camera/camera.py | 50 ++++++++++++------ .../test/sensors/test_opencv_distortion.py | 52 ++++++++++++++++++- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 7c36ba0ef7df..8b2cb012d0d3 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -204,8 +204,9 @@ def __init__(self, cfg: CameraCfg): # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = list() - # Guard so the lens-distortion image-size mismatch warning is emitted at most once. - self._warned_distortion_image_size: bool = False + # one-time lens-distortion warning guards, image-size mismatches keyed by authored size + self._warned_distortion_image_sizes: set[tuple[int, int]] = set() + self._warned_distortion_missing_intrinsics: bool = False # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # Renderer and render data — assigned in _initialize_impl. @@ -661,26 +662,43 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = # assumes square pixels and a centered principal point. raw_prim = sensor_prim.GetPrim() distortion_model = raw_prim.GetAttribute("omni:lensdistortion:model").Get() - if distortion_model: - prefix = f"omni:lensdistortion:{distortion_model}" - f_x = float(raw_prim.GetAttribute(f"{prefix}:fx").Get()) - f_y = float(raw_prim.GetAttribute(f"{prefix}:fy").Get()) - c_x = float(raw_prim.GetAttribute(f"{prefix}:cx").Get()) - c_y = float(raw_prim.GetAttribute(f"{prefix}:cy").Get()) - # the intrinsics are reported in the calibrated pixel space; warn once if the render - # resolution differs, since the reported matrix will not match the rendered pixels. + prefix = f"omni:lensdistortion:{distortion_model}" if distortion_model else None + # read the authored intrinsics; a prim may carry the model token without fx/fy/cx/cy + fx_val = fy_val = cx_val = cy_val = None + if prefix is not None: + fx_val = raw_prim.GetAttribute(f"{prefix}:fx").Get() + fy_val = raw_prim.GetAttribute(f"{prefix}:fy").Get() + cx_val = raw_prim.GetAttribute(f"{prefix}:cx").Get() + cy_val = raw_prim.GetAttribute(f"{prefix}:cy").Get() + if prefix is not None and None not in (fx_val, fy_val, cx_val, cy_val): + f_x, f_y, c_x, c_y = float(fx_val), float(fy_val), float(cx_val), float(cy_val) + # the intrinsics are reported in the calibrated pixel space; warn once per authored + # size if the render resolution differs, since the matrix will not match the pixels. image_size = raw_prim.GetAttribute(f"{prefix}:imageSize").Get() - if image_size is not None and (int(image_size[0]), int(image_size[1])) != (width, height): - if not self._warned_distortion_image_size: + if image_size is not None: + authored_size = (int(image_size[0]), int(image_size[1])) + if authored_size != (width, height) and authored_size not in self._warned_distortion_image_sizes: logger.warning( - "Camera lens-distortion 'imageSize' %s does not match the render resolution" - " (%d, %d). The reported intrinsic matrix is in the calibrated pixel space.", - (int(image_size[0]), int(image_size[1])), + "Camera prim '%s' (env %d) lens-distortion 'imageSize' %s does not match the" + " render resolution (%d, %d). The reported intrinsic matrix is in the" + " calibrated pixel space.", + raw_prim.GetPath(), + int(i), + authored_size, width, height, ) - self._warned_distortion_image_size = True + self._warned_distortion_image_sizes.add(authored_size) else: + # a model token without intrinsics falls back to the focal-length/aperture projection + if prefix is not None and not self._warned_distortion_missing_intrinsics: + logger.warning( + "Camera prim '%s' declares lens-distortion model '%s' but is missing one or more" + " fx/fy/cx/cy intrinsics; falling back to the focal-length/aperture projection.", + raw_prim.GetPath(), + distortion_model, + ) + self._warned_distortion_missing_intrinsics = True # get camera parameters # currently rendering does not use aperture offsets or vertical aperture focal_length = sensor_prim.GetFocalLengthAttr().Get() diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 4312ea213c16..daf3e6c62b8b 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -183,7 +183,8 @@ def _capture_state(env_ids=None, intrinsics_src=None, update_intrinsics=False): fake = Camera.__new__(Camera) fake._sensor_prims = [cam] fake._device = "cpu" - fake._warned_distortion_image_size = False + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False fake.cfg = SimpleNamespace(height=height, width=width) fake._resolve_env_ids_np = lambda env_ids: np.array([0]) fake._update_camera_state = _capture_state @@ -227,3 +228,52 @@ def test_readback_preserves_non_square_and_off_center(): # principal point is off-center (the old readback forced width/2, height/2) assert k[0, 2] != pytest.approx(width / 2) assert k[1, 2] != pytest.approx(height / 2) + + +def _camera_prim_with_model_token_only(): + """Build a camera prim that declares the distortion model token but authors no fx/fy/cx/cy.""" + stage = Usd.Stage.CreateInMemory() + cam = UsdGeom.Camera.Define(stage, "/Camera") + cam.GetPrim().CreateAttribute("omni:lensdistortion:model", Sdf.ValueTypeNames.Token).Set("opencvPinhole") + return stage, cam + + +def test_readback_missing_intrinsics_falls_back_to_focal_length(): + """A model token without fx/fy/cx/cy falls back to the focal-length projection instead of raising.""" + width, height = 640, 480 + _stage, cam = _camera_prim_with_model_token_only() + + k = _read_back_intrinsics(cam, width, height) + + # focal-length/aperture projection: square pixels, centered principal point + assert k[0, 0] == pytest.approx(k[1, 1]) + assert k[0, 2] == pytest.approx(width / 2) + assert k[1, 2] == pytest.approx(height / 2) + assert k[2, 2] == pytest.approx(1.0) + + +def test_readback_distinct_image_size_mismatches_each_warn(): + """Distinct authored image sizes across a camera's prims each warn, not only the first.""" + width, height = 320, 240 + _stage_a, cam_a = _camera_prim_with_pinhole_distortion(300.0, 300.0, 160.0, 120.0, 640, 480) + _stage_b, cam_b = _camera_prim_with_pinhole_distortion(300.0, 300.0, 160.0, 120.0, 1280, 720) + + fake = Camera.__new__(Camera) + fake._sensor_prims = [cam_a, cam_b] + fake._device = "cpu" + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False + fake.cfg = SimpleNamespace(height=height, width=width) + fake._resolve_env_ids_np = lambda env_ids: np.array([0, 1]) + fake._update_camera_state = lambda **kwargs: None + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + Camera._update_intrinsic_matrices(fake) + + assert fake._warned_distortion_image_sizes == {(640, 480), (1280, 720)} From f65bb7085b643e6053846e8bcf68a402b3f3c1ee Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 18:39:29 +0300 Subject: [PATCH 05/11] Flatten OpenCV intrinsic readback into a helper Extract the authored OpenCV lens-distortion readback out of the per-camera loop in Camera._update_intrinsic_matrices into a dedicated _read_authored_opencv_intrinsics helper. The loop body drops from four levels of nesting to a single authored-vs-fallback branch, keeping the warn-once guards and behavior unchanged. Also refresh the method docstring, which still claimed intrinsics always assume square pixels, and tidy the warning-guard comment. --- .../isaaclab/sensors/camera/camera.py | 113 +++++++++++------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 8b2cb012d0d3..ec20e9bd6996 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -13,7 +13,7 @@ import torch import warp as wp -from pxr import UsdGeom, UsdPhysics +from pxr import Usd, UsdGeom, UsdPhysics import isaaclab.sim as sim_utils import isaaclab.utils.sensors as sensor_utils @@ -204,7 +204,8 @@ def __init__(self, cfg: CameraCfg): # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = list() - # one-time lens-distortion warning guards, image-size mismatches keyed by authored size + # one-time guards so the lens-distortion readback warnings fire at most once + # (image-size mismatches keyed by authored size) self._warned_distortion_image_sizes: set[tuple[int, int]] = set() self._warned_distortion_missing_intrinsics: bool = False # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. @@ -636,10 +637,67 @@ def _create_buffers(self): self._update_poses() self._renderer.set_outputs(self._render_data, self._data.output) + def _read_authored_opencv_intrinsics( + self, prim: Usd.Prim, width: int, height: int, env_id: int + ) -> tuple[float, float, float, float] | None: + """Read the authored OpenCV lens-distortion intrinsics from a camera prim. + + Returns the calibrated ``(fx, fy, cx, cy)`` that the RTX/OVRTX renderer projects through when + the prim carries a complete OpenCV lens-distortion model, otherwise ``None`` so the caller can + fall back to the focal-length/aperture projection. Unlike that projection, these intrinsics may + be non-square (``fx != fy``) or off-center. A missing-intrinsics or image-size mismatch warns at + most once per camera. + + Args: + prim: The camera prim to read the authored intrinsics from. + width: The render width in pixels. + height: The render height in pixels. + env_id: The environment index, used only for warning messages. + + Returns: + The authored ``(fx, fy, cx, cy)`` in pixels, or ``None`` when no complete model is present. + """ + distortion_model = prim.GetAttribute("omni:lensdistortion:model").Get() + if not distortion_model: + return None + prefix = f"omni:lensdistortion:{distortion_model}" + # a prim may carry the model token without fx/fy/cx/cy + intrinsics = tuple(prim.GetAttribute(f"{prefix}:{name}").Get() for name in ("fx", "fy", "cx", "cy")) + if None in intrinsics: + # a model token without intrinsics falls back to the focal-length/aperture projection + if not self._warned_distortion_missing_intrinsics: + logger.warning( + "Camera prim '%s' declares lens-distortion model '%s' but is missing one or more" + " fx/fy/cx/cy intrinsics; falling back to the focal-length/aperture projection.", + prim.GetPath(), + distortion_model, + ) + self._warned_distortion_missing_intrinsics = True + return None + # the intrinsics are reported in the calibrated pixel space; warn once per authored size if the + # render resolution differs, since the matrix will not match the rendered pixels. + image_size = prim.GetAttribute(f"{prefix}:imageSize").Get() + if image_size is not None: + authored_size = (int(image_size[0]), int(image_size[1])) + if authored_size != (width, height) and authored_size not in self._warned_distortion_image_sizes: + logger.warning( + "Camera prim '%s' (env %d) lens-distortion 'imageSize' %s does not match the render" + " resolution (%d, %d). The reported intrinsic matrix is in the calibrated pixel space.", + prim.GetPath(), + env_id, + authored_size, + width, + height, + ) + self._warned_distortion_image_sizes.add(authored_size) + return tuple(float(value) for value in intrinsics) + def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = None): """Compute camera's matrix of intrinsic parameters. - Also called calibration matrix. This matrix works for linear depth images. We assume square pixels. + Also called calibration matrix. This matrix works for linear depth images. Without an authored + OpenCV lens-distortion model the readback assumes square pixels and a centered principal point; + when such a model is present its calibrated ``fx/fy/cx/cy`` are used instead. .. note:: The calibration matrix projects points in the 3D scene onto an imaginary screen of the camera. @@ -656,54 +714,17 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = sensor_prim = self._sensor_prims[int(i)] # get viewport parameters height, width = self.image_shape - # Prefer an authored OpenCV lens-distortion model when present: it carries the - # authoritative fx/fy/cx/cy that the RTX/OVRTX renderer projects through. These may be - # non-square (fx != fy) or off-center, unlike the focal-length/aperture readback which - # assumes square pixels and a centered principal point. - raw_prim = sensor_prim.GetPrim() - distortion_model = raw_prim.GetAttribute("omni:lensdistortion:model").Get() - prefix = f"omni:lensdistortion:{distortion_model}" if distortion_model else None - # read the authored intrinsics; a prim may carry the model token without fx/fy/cx/cy - fx_val = fy_val = cx_val = cy_val = None - if prefix is not None: - fx_val = raw_prim.GetAttribute(f"{prefix}:fx").Get() - fy_val = raw_prim.GetAttribute(f"{prefix}:fy").Get() - cx_val = raw_prim.GetAttribute(f"{prefix}:cx").Get() - cy_val = raw_prim.GetAttribute(f"{prefix}:cy").Get() - if prefix is not None and None not in (fx_val, fy_val, cx_val, cy_val): - f_x, f_y, c_x, c_y = float(fx_val), float(fy_val), float(cx_val), float(cy_val) - # the intrinsics are reported in the calibrated pixel space; warn once per authored - # size if the render resolution differs, since the matrix will not match the pixels. - image_size = raw_prim.GetAttribute(f"{prefix}:imageSize").Get() - if image_size is not None: - authored_size = (int(image_size[0]), int(image_size[1])) - if authored_size != (width, height) and authored_size not in self._warned_distortion_image_sizes: - logger.warning( - "Camera prim '%s' (env %d) lens-distortion 'imageSize' %s does not match the" - " render resolution (%d, %d). The reported intrinsic matrix is in the" - " calibrated pixel space.", - raw_prim.GetPath(), - int(i), - authored_size, - width, - height, - ) - self._warned_distortion_image_sizes.add(authored_size) + # Prefer an authored OpenCV lens-distortion model when present: it carries the authoritative + # fx/fy/cx/cy that the RTX/OVRTX renderer projects through, which may be non-square or off-center. + authored = self._read_authored_opencv_intrinsics(sensor_prim.GetPrim(), width, height, int(i)) + if authored is not None: + f_x, f_y, c_x, c_y = authored else: - # a model token without intrinsics falls back to the focal-length/aperture projection - if prefix is not None and not self._warned_distortion_missing_intrinsics: - logger.warning( - "Camera prim '%s' declares lens-distortion model '%s' but is missing one or more" - " fx/fy/cx/cy intrinsics; falling back to the focal-length/aperture projection.", - raw_prim.GetPath(), - distortion_model, - ) - self._warned_distortion_missing_intrinsics = True # get camera parameters # currently rendering does not use aperture offsets or vertical aperture focal_length = sensor_prim.GetFocalLengthAttr().Get() horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get() - # extract intrinsic parameters + # extract intrinsic parameters (square pixels, centered principal point) f_x = (width * focal_length) / horiz_aperture f_y = f_x c_x = width * 0.5 From d1d7974b1a9b735656602468402a7264c2713688 Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 18:39:44 +0300 Subject: [PATCH 06/11] Access camera distortion cfg field directly in spawn_camera The distortion field is defined on PinholeCameraCfg and inherited by FisheyeCameraCfg, the two cfg types spawn_camera accepts, so the defensive getattr lookup is unnecessary and inconsistent with the direct attribute access used elsewhere in the function. --- source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py index a1e6ccbf29ac..e3a8470d553b 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py @@ -204,7 +204,7 @@ def spawn_camera( # get attribute from the class prim.GetAttribute(prim_prop_name).Set(param_value) # author the OpenCV lens-distortion model (renderer-agnostic; RTX/OVRTX honors it natively) - if getattr(cfg, "distortion", None) is not None: + if cfg.distortion is not None: _author_opencv_distortion(prim, cfg.distortion) # return the prim return prim From 4234bfd4985383ca0c14e052cfe4705b6831a104 Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 18:42:49 +0300 Subject: [PATCH 07/11] Mark OpenCV distortion unit tests with unit marker The kit-less test_opencv_distortion.py carried only a skipif and no level marker, unlike every sibling in test/sensors/ (kit-less unit tests use pytest.mark.unit). Without the level marker the file is not categorized as a unit test in the JUnit report nor selected by '-m unit', so CI's unit lane skips it. Add pytest.mark.unit to the module-level pytestmark to match the convention. --- .../isaaclab/test/sensors/test_opencv_distortion.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index daf3e6c62b8b..5b1aaaf7c29a 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -29,10 +29,13 @@ _REQUIRED_MODULES = ("isaaclab", "pxr", "warp") _MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] -pytestmark = pytest.mark.skipif( - bool(_MISSING_MODULES), - reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", -) +pytestmark = [ + pytest.mark.unit, + pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", + ), +] if not _MISSING_MODULES: from pxr import Gf, Sdf, Usd, UsdGeom From 98bbb9c95a6b086118626aa0b49a8c27e05b83a4 Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 18:52:29 +0300 Subject: [PATCH 08/11] Hoist loop-invariant image_shape out of intrinsics loop The (height, width) viewport read is constant across the per-env loop in _update_intrinsic_matrices; compute it once before the loop rather than on every iteration. --- source/isaaclab/isaaclab/sensors/camera/camera.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index ec20e9bd6996..6cf54a3406a2 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -708,12 +708,12 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = return intrinsic_matrices = np.zeros((len(env_ids_np), 3, 3), dtype=np.float32) + # viewport parameters are shared by every camera prim of this sensor + height, width = self.image_shape # iterate over all cameras for matrix_id, i in enumerate(env_ids_np): # Get corresponding sensor prim sensor_prim = self._sensor_prims[int(i)] - # get viewport parameters - height, width = self.image_shape # Prefer an authored OpenCV lens-distortion model when present: it carries the authoritative # fx/fy/cx/cy that the RTX/OVRTX renderer projects through, which may be non-square or off-center. authored = self._read_authored_opencv_intrinsics(sensor_prim.GetPrim(), width, height, int(i)) From 2814b49117bc399bf1c590c67319251affebd885 Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 18:52:29 +0300 Subject: [PATCH 09/11] Clarify OpenCV distortion docs and Newton renderer warning Describe the Camera.data intrinsics change in the changelog without cross-referencing a private method. Add reciprocal See Also notes between FisheyeCameraCfg and OpenCvFisheyeDistortionCfg, and mention the optional OpenCV calibration in the sensors spawner module docstring. Broaden the Newton renderer warning to state that a non-square fx, the principal point, and the distortion coefficients are all dropped (Newton derives one field of view from fy), not only the distortion. --- .../changelog.d/opencv-lens-distortion-cameras.minor.rst | 9 ++++----- .../isaaclab/isaaclab/sim/spawners/sensors/__init__.py | 3 ++- .../isaaclab/sim/spawners/sensors/sensors_cfg.py | 8 ++++++++ .../isaaclab_newton/renderers/newton_warp_renderer.py | 5 +++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst index 603804a2e5c4..f3f574447f04 100644 --- a/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst +++ b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst @@ -10,8 +10,7 @@ Added Changed ^^^^^^^ -* Changed :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices` to read the authored - ``fx/fy/cx/cy`` when an OpenCV lens-distortion model is present, so - :attr:`~isaaclab.sensors.camera.Camera.data` intrinsics reflect a real calibration (non-square - pixels or an off-center principal point) instead of assuming ``fx == fy`` and a centered principal - point. +* Changed the reconstructed :attr:`~isaaclab.sensors.camera.Camera.data` intrinsic matrices to use the + authored OpenCV ``fx/fy/cx/cy`` when a lens-distortion model is present, so they reflect a real + calibration (non-square pixels or an off-center principal point) instead of assuming ``fx == fy`` and + a centered principal point. diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py index 49249f9b6eb8..55a40ee19ffd 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py @@ -7,7 +7,8 @@ Currently, the following sensors are supported: -* Camera: A USD camera prim with settings for pinhole or fisheye projections. +* Camera: A USD camera prim with settings for pinhole or fisheye projections, optionally carrying an + OpenCV lens-distortion calibration. """ diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py index 82f0274aedd6..dccf09b2e0b1 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py @@ -111,6 +111,10 @@ class OpenCvFisheyeDistortionCfg(OpenCvDistortionCfg): """OpenCV fisheye lens-distortion model. Corresponds to ``OmniLensDistortionOpenCvFisheyeAPI`` under the RTX/OVRTX renderer. + + See Also: + :class:`FisheyeCameraCfg` for the USD ``fisheyePolynomial`` projection, an alternative fisheye + model authored directly on the camera rather than as an OpenCV ``fx/fy/cx/cy`` calibration. """ model: str = "opencvFisheye" @@ -296,6 +300,10 @@ class FisheyeCameraCfg(PinholeCameraCfg): function. .. _fish-eye camera: https://en.wikipedia.org/wiki/Fisheye_lens + + See Also: + :class:`OpenCvFisheyeDistortionCfg` for an OpenCV ``fx/fy/cx/cy`` + distortion-coefficient + fisheye calibration applied via the :attr:`~PinholeCameraCfg.distortion` field. """ func: Callable | str = "{DIR}.sensors:spawn_camera" diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 2a031359e51e..72d02c3ca6e8 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -392,8 +392,9 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: if getattr(spawn, "distortion", None) is not None: logger.warning( "OpenCV lens distortion is set on the camera cfg but is not yet applied by the Newton" - " renderer; the camera will render undistorted. Use the RTX/OVRTX renderer to apply" - " the distortion model." + " renderer: it derives a single field of view from fy, so the distortion coefficients," + " the principal point, and a non-square fx are ignored and the camera renders as a" + " centered, square-pixel pinhole. Use the RTX/OVRTX renderer to apply the full model." ) if spec.cfg.isp_cfg is None: return From e46ea2ba4be69acbbb8c97373868ba05fb6b1f3e Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 20:35:18 +0300 Subject: [PATCH 10/11] Skip only distortion cameras in set_intrinsic_matrices A camera with an authored OpenCV lens-distortion model owns its fx/fy/cx/cy through the omni:lensdistortion:* calibration, which the readback prefers; set_intrinsic_matrices only writes the square-pixel, centered focal-length/ aperture attributes and cannot express it. Skip such cameras per entry (with a warning) and leave their calibration untouched, while still updating any ordinary cameras selected in the same call. Regression-tested with a mixed distortion/plain batch. --- .../isaaclab/sensors/camera/camera.py | 29 +++++++-- .../test/sensors/test_opencv_distortion.py | 62 +++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 6cf54a3406a2..6470df7d94f1 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -281,11 +281,21 @@ def set_intrinsic_matrices( i.e. has square pixels, and the optical center is centered at the camera eye. If this assumption is not true in the input intrinsic matrix, then the camera will not set up correctly. + .. note:: + + Cameras carrying an OpenCV lens-distortion model are skipped (with a warning): their + ``fx/fy/cx/cy`` are fixed at spawn through the + :attr:`~isaaclab.sim.spawners.sensors.PinholeCameraCfg.distortion` cfg and cannot be overridden + here. Any other selected cameras in the same call are still updated. + Args: matrices: The intrinsic matrices for the camera. Shape is (N, 3, 3). focal_length: Perspective focal length (in cm) used to calculate pixel size. Defaults to None. If None, focal_length will be calculated 1 / width. env_ids: A sensor ids to manipulate. Defaults to None, which means all sensor indices. + + Raises: + TypeError: If ``matrices`` is not a :class:`torch.Tensor` or a Warp array. """ if isinstance(matrices, torch.Tensor): if not matrices.is_contiguous(): @@ -305,15 +315,21 @@ def set_intrinsic_matrices( if matrices.ndim == 2: matrices = matrices[None, ...] # iterate over env_ids + height, width = self.image_shape + skipped_distortion = False for i, intrinsic_matrix in zip(env_ids_np, matrices): - height, width = self.image_shape + # change data for corresponding camera index + sensor_prim = self._sensor_prims[i] + # A camera with an authored OpenCV lens-distortion model owns its fx/fy/cx/cy through the + # ``omni:lensdistortion:*`` calibration, which this square-pixel, centered focal-length/aperture + # write cannot express, so skip that camera and leave its calibration untouched. + if sensor_prim.GetPrim().GetAttribute("omni:lensdistortion:model").Get(): + skipped_distortion = True + continue params = sensor_utils.convert_camera_intrinsics_to_usd( intrinsic_matrix=intrinsic_matrix.reshape(-1), height=height, width=width, focal_length=focal_length ) - - # change data for corresponding camera index - sensor_prim = self._sensor_prims[i] # set parameters for camera for param_name, param_value in params.items(): # convert to camel case (CC) @@ -325,6 +341,11 @@ def set_intrinsic_matrices( param_value = float(param_value) # set value using pure USD API param_attr().Set(param_value) + if skipped_distortion: + logger.warning( + "set_intrinsic_matrices() skipped one or more cameras configured with an OpenCV" + " lens-distortion model; their intrinsics are fixed at spawn via the 'distortion' cfg." + ) # update the internal buffers self._update_intrinsic_matrices(env_ids_np) diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py index 5b1aaaf7c29a..832eaf956390 100644 --- a/source/isaaclab/test/sensors/test_opencv_distortion.py +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -21,6 +21,7 @@ from __future__ import annotations import importlib.util +import logging from types import SimpleNamespace import numpy as np @@ -38,6 +39,8 @@ ] if not _MISSING_MODULES: + import torch + from pxr import Gf, Sdf, Usd, UsdGeom import isaaclab.sim as sim_utils @@ -280,3 +283,62 @@ def test_readback_distinct_image_size_mismatches_each_warn(): Camera._update_intrinsic_matrices(fake) assert fake._warned_distortion_image_sizes == {(640, 480), (1280, 720)} + + +def test_set_intrinsic_matrices_skips_only_distortion_cameras_in_batch(): + """In a mixed batch only the distortion camera is skipped (with a warning); a plain camera is updated. + + The authored ``omni:lensdistortion:*`` fx/fy/cx/cy are the readback's source of truth, so the + focal-length/aperture write would be discarded for a distortion camera. Skipping the whole call would + also drop ordinary selected cameras; only the distortion entries must be left untouched. + """ + width, height = 640, 480 + _stage_d, distortion_cam = _camera_prim_with_pinhole_distortion(339.0, 338.0, 323.0, 250.0, width, height) + plain_stage = Usd.Stage.CreateInMemory() + plain_cam = UsdGeom.Camera.Define(plain_stage, "/PlainCamera") + + captured = {} + + def _capture_state(env_ids=None, intrinsics_src=None, update_intrinsics=False): + captured["K"] = intrinsics_src.numpy() + + fake = Camera.__new__(Camera) + fake._sensor_prims = [distortion_cam, plain_cam] + fake._device = "cpu" + fake.cfg = SimpleNamespace(height=height, width=width) + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False + fake._resolve_env_ids_np = lambda env_ids: np.array([0, 1]) + fake._update_camera_state = _capture_state + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + messages: list[str] = [] + handler = logging.Handler() + handler.emit = lambda record: messages.append(record.getMessage()) + cam_logger = logging.getLogger("isaaclab.sensors.camera.camera") + cam_logger.addHandler(handler) + # row 0 targets the distortion camera (skipped); row 1 recalibrates the plain camera to fx = fy = 500 + requested = torch.tensor( + [ + [[999.0, 0.0, 111.0], [0.0, 999.0, 222.0], [0.0, 0.0, 1.0]], + [[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]], + ], + dtype=torch.float32, + ) + try: + Camera.set_intrinsic_matrices(fake, requested, env_ids=[0, 1]) + finally: + cam_logger.removeHandler(handler) + + k = captured["K"] + # the distortion camera keeps its authored calibration (skipped, request ignored) + assert k[0, 0, 0] == pytest.approx(339.0, abs=1e-2) + # the plain camera reflects the requested focal length (updated, not over-skipped) + assert k[1, 0, 0] == pytest.approx(500.0, abs=1e-2) + assert any("skipped" in message.lower() for message in messages) From f6b9305cb30a270947675151274d2ae5660b8c27 Mon Sep 17 00:00:00 2001 From: lbenhorin Date: Sun, 19 Jul 2026 20:35:18 +0300 Subject: [PATCH 11/11] Add OVRTX fisheye lens-distortion render test Render the OpenCV fisheye model and an undistorted pinhole through OVRTX and assert the frames differ well beyond render noise, guarding that the renderer honors the OmniLensDistortionOpenCvFisheyeAPI schema. --- .../test_camera_opencv_distortion_ovrtx.py | 74 ++++++++++++++++--- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py index d5f085b27fc8..0108b8cb429c 100644 --- a/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py +++ b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py @@ -20,6 +20,8 @@ OVRTX render context applies one lens model, so the two passes must not share a process context. """ +from __future__ import annotations + import importlib.util import numpy as np @@ -45,7 +47,12 @@ from isaaclab.scene import InteractiveScene, InteractiveSceneCfg from isaaclab.sensors import Camera, CameraCfg from isaaclab.sim import SimulationCfg - from isaaclab.sim.spawners.sensors.sensors_cfg import OpenCvPinholeDistortionCfg, PinholeCameraCfg + from isaaclab.sim.spawners.sensors.sensors_cfg import ( + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + ) from isaaclab.utils.configclass import configclass from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix @@ -53,11 +60,13 @@ WIDTH, HEIGHT = 640, 480 WARMUP_STEPS = 4 -# Real SO-101 wrist-camera calibration (fx != fy, off-center principal point). +# Example real-world OpenCV pinhole calibration (fx != fy, off-center principal point). _CALIB = dict(fx=339.26592887, fy=338.82010626, cx=323.55809091, cy=250.27360914) _COEFFS = dict(k1=0.07702322, k2=-0.13605453, k3=0.05163219, p1=-0.00024938, p2=-0.00175006) # scale the (mild) real coefficients so the barrel effect is unambiguous in the assertion _K_SCALE = 15.0 +# OpenCV fisheye (equidistant) coefficients; the base fisheye projection alone differs strongly from pinhole +_FISHEYE_COEFFS = dict(k1=0.1, k2=-0.05, k3=0.0, k4=0.0) _CAM_EYE = (0.0, 0.0, 2.5) _CAM_TARGET = (1.75, 0.0, 0.0) @@ -88,7 +97,27 @@ class _DistortionSceneCfg(InteractiveSceneCfg): ) -def _render_grid(apply_lens_distortion: bool, device: str) -> tuple[np.ndarray, np.ndarray]: +def _pinhole_distortion(apply_lens_distortion: bool) -> OpenCvPinholeDistortionCfg: + """Pinhole OpenCV calibration with the (scaled) SO-101 radial/tangential coefficients.""" + return OpenCvPinholeDistortionCfg( + image_size=(WIDTH, HEIGHT), + apply_lens_distortion=apply_lens_distortion, + **_CALIB, + **{name: value * _K_SCALE for name, value in _COEFFS.items()}, + ) + + +def _fisheye_distortion(apply_lens_distortion: bool) -> OpenCvFisheyeDistortionCfg: + """Fisheye OpenCV calibration reusing the SO-101 intrinsics with fisheye coefficients.""" + return OpenCvFisheyeDistortionCfg( + image_size=(WIDTH, HEIGHT), + apply_lens_distortion=apply_lens_distortion, + **_CALIB, + **_FISHEYE_COEFFS, + ) + + +def _render_grid(distortion: OpenCvDistortionCfg, device: str) -> tuple[np.ndarray, np.ndarray]: """Render the ground-plane grid through an OpenCV-calibrated OVRTX camera; return ``(rgb, K)``.""" sim_utils.create_new_stage() sim = sim_utils.SimulationContext( @@ -101,12 +130,6 @@ def _render_grid(apply_lens_distortion: bool, device: str) -> tuple[np.ndarray, create_rotation_matrix_from_view(torch.tensor([_CAM_EYE]), torch.tensor([_CAM_TARGET]), up_axis="Z") )[0].tolist() ) - distortion = OpenCvPinholeDistortionCfg( - image_size=(WIDTH, HEIGHT), - apply_lens_distortion=apply_lens_distortion, - **_CALIB, - **{name: value * _K_SCALE for name, value in _COEFFS.items()}, - ) camera = Camera( CameraCfg( prim_path="/World/envs/env_.*/Camera", @@ -139,8 +162,8 @@ def _render_grid(apply_lens_distortion: bool, device: str) -> tuple[np.ndarray, @_SKIP_MISSING_OVRTX def test_opencv_distortion_changes_ovrtx_render(device): """OVRTX must render the distorted and zero-coefficient cameras meaningfully differently.""" - distorted, _ = _render_grid(apply_lens_distortion=True, device=device) - reference, _ = _render_grid(apply_lens_distortion=False, device=device) + distorted, _ = _render_grid(_pinhole_distortion(True), device=device) + reference, _ = _render_grid(_pinhole_distortion(False), device=device) assert distorted.shape == (HEIGHT, WIDTH, 3) # both frames render content (not degenerate) @@ -156,7 +179,7 @@ def test_opencv_distortion_changes_ovrtx_render(device): @_SKIP_MISSING_OVRTX def test_opencv_distortion_intrinsics_match_authored_ovrtx(device): """The OVRTX camera reports intrinsics matching the authored, non-square, off-center calibration.""" - _rgb, k = _render_grid(apply_lens_distortion=True, device=device) + _rgb, k = _render_grid(_pinhole_distortion(True), device=device) assert k[0, 0] == pytest.approx(_CALIB["fx"], abs=1e-2) assert k[1, 1] == pytest.approx(_CALIB["fy"], abs=1e-2) @@ -165,3 +188,30 @@ def test_opencv_distortion_intrinsics_match_authored_ovrtx(device): # not the stock fx == fy / centered-principal-point collapse assert k[0, 0] != k[1, 1] assert k[0, 2] != pytest.approx(WIDTH / 2) + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_fisheye_distortion_renders_through_ovrtx(device): + """OVRTX honors the OpenCV fisheye schema: its render differs meaningfully from the pinhole projection. + + The same calibrated camera is rendered under the OpenCV fisheye model and under an undistorted + pinhole. The fisheye equidistant projection bends the straight grid, so the two frames must differ + well beyond render noise, and the reported intrinsics must still match the authored calibration. + """ + fisheye, k = _render_grid(_fisheye_distortion(True), device=device) + pinhole, _ = _render_grid(_pinhole_distortion(False), device=device) + + assert fisheye.shape == (HEIGHT, WIDTH, 3) + # both frames render content (not degenerate) + assert fisheye.std() > 1.0 + assert pinhole.std() > 1.0 + # OVRTX applied the fisheye projection: it differs from the pinhole render well beyond render noise + mean_abs_diff = np.abs(fisheye.astype(np.float32) - pinhole.astype(np.float32)).mean() + assert mean_abs_diff > 2.0, f"fisheye vs pinhole differ by only {mean_abs_diff:.3f}/255" + # the fisheye camera still reports the authored, non-square, off-center calibration + assert k[0, 0] == pytest.approx(_CALIB["fx"], abs=1e-2) + assert k[1, 1] == pytest.approx(_CALIB["fy"], abs=1e-2) + assert k[0, 2] == pytest.approx(_CALIB["cx"], abs=1e-2) + assert k[1, 2] == pytest.approx(_CALIB["cy"], abs=1e-2)