diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index da5d955d0f9d..d5c40dd7057a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -125,6 +125,7 @@ Guidelines for modifications: * Krishna Lakhi * Lin He * Lionel Gulich +* Lior Ben Horin * Lorenz Wellhausen * Lotus Li * Louis Le Lay diff --git a/docs/source/api/lab/isaaclab.sim.spawners.rst b/docs/source/api/lab/isaaclab.sim.spawners.rst index ce82c003e8e5..fd9e5231217f 100644 --- a/docs/source/api/lab/isaaclab.sim.spawners.rst +++ b/docs/source/api/lab/isaaclab.sim.spawners.rst @@ -211,6 +211,9 @@ Sensors PinholeCameraCfg FisheyeCameraCfg + OpenCvDistortionCfg + OpenCvPinholeDistortionCfg + OpenCvFisheyeDistortionCfg .. autofunction:: spawn_camera @@ -222,6 +225,18 @@ Sensors :members: :exclude-members: __init__, func +.. autoclass:: OpenCvDistortionCfg + :members: + :exclude-members: __init__, func + +.. autoclass:: OpenCvPinholeDistortionCfg + :members: + :exclude-members: __init__, func + +.. autoclass:: OpenCvFisheyeDistortionCfg + :members: + :exclude-members: __init__, func + From Files ---------- diff --git a/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst new file mode 100644 index 000000000000..f3f574447f04 --- /dev/null +++ b/source/isaaclab/changelog.d/opencv-lens-distortion-cameras.minor.rst @@ -0,0 +1,16 @@ +Added +^^^^^ + +* Added :class:`~isaaclab.sim.spawners.sensors.OpenCvPinholeDistortionCfg` and + :class:`~isaaclab.sim.spawners.sensors.OpenCvFisheyeDistortionCfg` and a ``distortion`` field on + :class:`~isaaclab.sim.spawners.sensors.PinholeCameraCfg`, letting a camera carry an OpenCV + ``fx/fy/cx/cy`` + distortion-coefficient calibration. The model is authored on the camera prim as + the ``omni:lensdistortion:*`` USD API, which the RTX/OVRTX renderer honors natively. + +Changed +^^^^^^^ + +* Changed the reconstructed :attr:`~isaaclab.sensors.camera.Camera.data` intrinsic matrices to use the + authored OpenCV ``fx/fy/cx/cy`` when a lens-distortion model is present, so they reflect a real + calibration (non-square pixels or an off-center principal point) instead of assuming ``fx == fy`` and + a centered principal point. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 58cf4423eaf5..6470df7d94f1 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -13,7 +13,7 @@ import torch import warp as wp -from pxr import UsdGeom, UsdPhysics +from pxr import Usd, UsdGeom, UsdPhysics import isaaclab.sim as sim_utils import isaaclab.utils.sensors as sensor_utils @@ -204,6 +204,10 @@ def __init__(self, cfg: CameraCfg): # UsdGeom Camera prim for the sensor self._sensor_prims: list[UsdGeom.Camera] = list() + # one-time guards so the lens-distortion readback warnings fire at most once + # (image-size mismatches keyed by authored size) + self._warned_distortion_image_sizes: set[tuple[int, int]] = set() + self._warned_distortion_missing_intrinsics: bool = False # Allocated in :meth:`_create_buffers` once the renderer's output contract is known. self._data: CameraData | None = None # Renderer and render data — assigned in _initialize_impl. @@ -277,11 +281,21 @@ def set_intrinsic_matrices( i.e. has square pixels, and the optical center is centered at the camera eye. If this assumption is not true in the input intrinsic matrix, then the camera will not set up correctly. + .. note:: + + Cameras carrying an OpenCV lens-distortion model are skipped (with a warning): their + ``fx/fy/cx/cy`` are fixed at spawn through the + :attr:`~isaaclab.sim.spawners.sensors.PinholeCameraCfg.distortion` cfg and cannot be overridden + here. Any other selected cameras in the same call are still updated. + Args: matrices: The intrinsic matrices for the camera. Shape is (N, 3, 3). focal_length: Perspective focal length (in cm) used to calculate pixel size. Defaults to None. If None, focal_length will be calculated 1 / width. env_ids: A sensor ids to manipulate. Defaults to None, which means all sensor indices. + + Raises: + TypeError: If ``matrices`` is not a :class:`torch.Tensor` or a Warp array. """ if isinstance(matrices, torch.Tensor): if not matrices.is_contiguous(): @@ -301,15 +315,21 @@ def set_intrinsic_matrices( if matrices.ndim == 2: matrices = matrices[None, ...] # iterate over env_ids + height, width = self.image_shape + skipped_distortion = False for i, intrinsic_matrix in zip(env_ids_np, matrices): - height, width = self.image_shape + # change data for corresponding camera index + sensor_prim = self._sensor_prims[i] + # A camera with an authored OpenCV lens-distortion model owns its fx/fy/cx/cy through the + # ``omni:lensdistortion:*`` calibration, which this square-pixel, centered focal-length/aperture + # write cannot express, so skip that camera and leave its calibration untouched. + if sensor_prim.GetPrim().GetAttribute("omni:lensdistortion:model").Get(): + skipped_distortion = True + continue params = sensor_utils.convert_camera_intrinsics_to_usd( intrinsic_matrix=intrinsic_matrix.reshape(-1), height=height, width=width, focal_length=focal_length ) - - # change data for corresponding camera index - sensor_prim = self._sensor_prims[i] # set parameters for camera for param_name, param_value in params.items(): # convert to camel case (CC) @@ -321,6 +341,11 @@ def set_intrinsic_matrices( param_value = float(param_value) # set value using pure USD API param_attr().Set(param_value) + if skipped_distortion: + logger.warning( + "set_intrinsic_matrices() skipped one or more cameras configured with an OpenCV" + " lens-distortion model; their intrinsics are fixed at spawn via the 'distortion' cfg." + ) # update the internal buffers self._update_intrinsic_matrices(env_ids_np) @@ -633,10 +658,67 @@ def _create_buffers(self): self._update_poses() self._renderer.set_outputs(self._render_data, self._data.output) + def _read_authored_opencv_intrinsics( + self, prim: Usd.Prim, width: int, height: int, env_id: int + ) -> tuple[float, float, float, float] | None: + """Read the authored OpenCV lens-distortion intrinsics from a camera prim. + + Returns the calibrated ``(fx, fy, cx, cy)`` that the RTX/OVRTX renderer projects through when + the prim carries a complete OpenCV lens-distortion model, otherwise ``None`` so the caller can + fall back to the focal-length/aperture projection. Unlike that projection, these intrinsics may + be non-square (``fx != fy``) or off-center. A missing-intrinsics or image-size mismatch warns at + most once per camera. + + Args: + prim: The camera prim to read the authored intrinsics from. + width: The render width in pixels. + height: The render height in pixels. + env_id: The environment index, used only for warning messages. + + Returns: + The authored ``(fx, fy, cx, cy)`` in pixels, or ``None`` when no complete model is present. + """ + distortion_model = prim.GetAttribute("omni:lensdistortion:model").Get() + if not distortion_model: + return None + prefix = f"omni:lensdistortion:{distortion_model}" + # a prim may carry the model token without fx/fy/cx/cy + intrinsics = tuple(prim.GetAttribute(f"{prefix}:{name}").Get() for name in ("fx", "fy", "cx", "cy")) + if None in intrinsics: + # a model token without intrinsics falls back to the focal-length/aperture projection + if not self._warned_distortion_missing_intrinsics: + logger.warning( + "Camera prim '%s' declares lens-distortion model '%s' but is missing one or more" + " fx/fy/cx/cy intrinsics; falling back to the focal-length/aperture projection.", + prim.GetPath(), + distortion_model, + ) + self._warned_distortion_missing_intrinsics = True + return None + # the intrinsics are reported in the calibrated pixel space; warn once per authored size if the + # render resolution differs, since the matrix will not match the rendered pixels. + image_size = prim.GetAttribute(f"{prefix}:imageSize").Get() + if image_size is not None: + authored_size = (int(image_size[0]), int(image_size[1])) + if authored_size != (width, height) and authored_size not in self._warned_distortion_image_sizes: + logger.warning( + "Camera prim '%s' (env %d) lens-distortion 'imageSize' %s does not match the render" + " resolution (%d, %d). The reported intrinsic matrix is in the calibrated pixel space.", + prim.GetPath(), + env_id, + authored_size, + width, + height, + ) + self._warned_distortion_image_sizes.add(authored_size) + return tuple(float(value) for value in intrinsics) + def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = None): """Compute camera's matrix of intrinsic parameters. - Also called calibration matrix. This matrix works for linear depth images. We assume square pixels. + Also called calibration matrix. This matrix works for linear depth images. Without an authored + OpenCV lens-distortion model the readback assumes square pixels and a centered principal point; + when such a model is present its calibrated ``fx/fy/cx/cy`` are used instead. .. note:: The calibration matrix projects points in the 3D scene onto an imaginary screen of the camera. @@ -647,22 +729,27 @@ def _update_intrinsic_matrices(self, env_ids: Sequence[int] | wp.array | None = return intrinsic_matrices = np.zeros((len(env_ids_np), 3, 3), dtype=np.float32) + # viewport parameters are shared by every camera prim of this sensor + height, width = self.image_shape # iterate over all cameras for matrix_id, i in enumerate(env_ids_np): # Get corresponding sensor prim sensor_prim = self._sensor_prims[int(i)] - # get camera parameters - # currently rendering does not use aperture offsets or vertical aperture - focal_length = sensor_prim.GetFocalLengthAttr().Get() - horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get() - - # get viewport parameters - height, width = self.image_shape - # extract intrinsic parameters - f_x = (width * focal_length) / horiz_aperture - f_y = f_x - c_x = width * 0.5 - c_y = height * 0.5 + # Prefer an authored OpenCV lens-distortion model when present: it carries the authoritative + # fx/fy/cx/cy that the RTX/OVRTX renderer projects through, which may be non-square or off-center. + authored = self._read_authored_opencv_intrinsics(sensor_prim.GetPrim(), width, height, int(i)) + if authored is not None: + f_x, f_y, c_x, c_y = authored + else: + # get camera parameters + # currently rendering does not use aperture offsets or vertical aperture + focal_length = sensor_prim.GetFocalLengthAttr().Get() + horiz_aperture = sensor_prim.GetHorizontalApertureAttr().Get() + # extract intrinsic parameters (square pixels, centered principal point) + f_x = (width * focal_length) / horiz_aperture + f_y = f_x + c_x = width * 0.5 + c_y = height * 0.5 # create intrinsic matrix for depth linear intrinsic_matrices[matrix_id, 0, 0] = f_x intrinsic_matrices[matrix_id, 0, 2] = c_x diff --git a/source/isaaclab/isaaclab/sim/__init__.pyi b/source/isaaclab/isaaclab/sim/__init__.pyi index f145f9879daf..8496bca7f39f 100644 --- a/source/isaaclab/isaaclab/sim/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/__init__.pyi @@ -137,6 +137,9 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", "spawn_capsule", @@ -332,6 +335,9 @@ from .spawners import ( MjcfFileCfg, MultiAssetSpawnerCfg, MultiUsdFileCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, PhysicsMaterialCfg, PinholeCameraCfg, PreviewSurfaceCfg, diff --git a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi index c936d166ba3c..c2d19f0cbde4 100644 --- a/source/isaaclab/isaaclab/sim/spawners/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/__init__.pyi @@ -54,6 +54,9 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", "spawn_capsule", @@ -126,7 +129,16 @@ from .meshes import ( MeshRectangleCfg, MeshSphereCfg, ) -from .sensors import spawn_camera, spawn_sensor_frame, FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg +from .sensors import ( + spawn_camera, + spawn_sensor_frame, + FisheyeCameraCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + SensorFrameCfg, +) from .shapes import ( spawn_capsule, spawn_cone, diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py index 49249f9b6eb8..55a40ee19ffd 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py @@ -7,7 +7,8 @@ Currently, the following sensors are supported: -* Camera: A USD camera prim with settings for pinhole or fisheye projections. +* Camera: A USD camera prim with settings for pinhole or fisheye projections, optionally carrying an + OpenCV lens-distortion calibration. """ diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi index ba5b96a44c7d..20c18ff9bfa8 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.pyi @@ -7,9 +7,19 @@ __all__ = [ "spawn_camera", "spawn_sensor_frame", "FisheyeCameraCfg", + "OpenCvDistortionCfg", + "OpenCvFisheyeDistortionCfg", + "OpenCvPinholeDistortionCfg", "PinholeCameraCfg", "SensorFrameCfg", ] from .sensors import spawn_camera, spawn_sensor_frame -from .sensors_cfg import FisheyeCameraCfg, PinholeCameraCfg, SensorFrameCfg +from .sensors_cfg import ( + FisheyeCameraCfg, + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + SensorFrameCfg, +) diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py index db68d21d8a90..e3a8470d553b 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py @@ -8,7 +8,7 @@ import logging from typing import TYPE_CHECKING -from pxr import Sdf, Usd +from pxr import Gf, Sdf, Usd from isaaclab.sim.utils import change_prim_property, clone, create_prim, get_current_stage from isaaclab.utils import to_camel_case @@ -48,6 +48,65 @@ """ +# OpenCV lens-distortion models authored as the ``omni:lensdistortion:*`` USD API. The RTX/OVRTX +# renderer honors these attributes natively; they are read back into ``camera.data.intrinsic_matrices`` +# by :meth:`~isaaclab.sensors.camera.Camera._update_intrinsic_matrices`. +_OPENCV_DISTORTION_API_SCHEMAS = { + "opencvPinhole": "OmniLensDistortionOpenCvPinholeAPI", + "opencvFisheye": "OmniLensDistortionOpenCvFisheyeAPI", +} +"""Maps an OpenCV distortion model discriminator to its applied USD API schema name.""" + +_OPENCV_DISTORTION_COEFFS = { + "opencvPinhole": ("k1", "k2", "k3", "k4", "k5", "k6", "p1", "p2", "s1", "s2", "s3", "s4"), + "opencvFisheye": ("k1", "k2", "k3", "k4"), +} +"""Maps an OpenCV distortion model discriminator to its distortion-coefficient field names.""" + + +def _author_opencv_distortion(prim: Usd.Prim, cfg: sensors_cfg.OpenCvDistortionCfg) -> None: + """Author an OpenCV lens-distortion model on a camera prim as the ``omni:lensdistortion:*`` API. + + The attributes are authored explicitly (not through the generic camelCase loop of + :func:`spawn_camera`) because their names are namespaced (e.g. ``omni:lensdistortion:opencvPinhole:k1``) + and cannot be produced by :func:`~isaaclab.utils.to_camel_case`. Applying the schema only edits prim + metadata, so it survives ``stage.ExportToString()`` and does not require the schema to be registered. + + Args: + prim: The camera prim to author the distortion model on. + cfg: The OpenCV distortion configuration. + + Raises: + ValueError: If the distortion ``model`` is not a supported OpenCV model. + """ + if cfg.model not in _OPENCV_DISTORTION_API_SCHEMAS: + raise ValueError( + f"Unsupported OpenCV distortion model: '{cfg.model}'. Supported models are:" + f" {list(_OPENCV_DISTORTION_API_SCHEMAS)}." + ) + prefix = f"omni:lensdistortion:{cfg.model}" + + # apply the schema and set the model discriminator token + prim.AddAppliedSchema(_OPENCV_DISTORTION_API_SCHEMAS[cfg.model]) + + def _set_attr(name: str, type_name: Sdf.ValueTypeName, value) -> None: + attr = prim.GetAttribute(name) or prim.CreateAttribute(name, type_name) + attr.Set(value) + + _set_attr("omni:lensdistortion:model", Sdf.ValueTypeNames.Token, cfg.model) + _set_attr( + f"{prefix}:imageSize", + Sdf.ValueTypeNames.Int2, + Gf.Vec2i(int(cfg.image_size[0]), int(cfg.image_size[1])), + ) + for name in ("fx", "fy", "cx", "cy"): + _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, float(getattr(cfg, name))) + # coefficients are muted (authored as zero) unless apply_lens_distortion is set + for name in _OPENCV_DISTORTION_COEFFS[cfg.model]: + value = float(getattr(cfg, name)) if cfg.apply_lens_distortion else 0.0 + _set_attr(f"{prefix}:{name}", Sdf.ValueTypeNames.Float, value) + + @clone def spawn_camera( prim_path: str, @@ -119,6 +178,7 @@ def spawn_camera( "semantic_tags", "from_intrinsic_matrix", "spawn_path", + "distortion", ] # get camera prim prim = stage.GetPrimAtPath(prim_path) @@ -143,6 +203,9 @@ def spawn_camera( prim_prop_name = to_camel_case(param_name, to="cC") # get attribute from the class prim.GetAttribute(prim_prop_name).Set(param_value) + # author the OpenCV lens-distortion model (renderer-agnostic; RTX/OVRTX honors it natively) + if cfg.distortion is not None: + _author_opencv_distortion(prim, cfg.distortion) # return the prim return prim diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py index 6a1741675074..dccf09b2e0b1 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import MISSING from typing import Literal import isaaclab.utils.sensors as sensor_utils @@ -13,6 +14,124 @@ from isaaclab.utils.configclass import configclass +@configclass +class OpenCvDistortionCfg: + """Base configuration for an OpenCV lens-distortion model carried on a camera cfg. + + The distortion model is renderer-agnostic: it is stored on the camera spawn configuration + and each renderer decides how to consume it. Under the RTX/OVRTX renderer the fields are + authored as the ``omni:lensdistortion:*`` USD API, which the renderer honors natively. The + Newton renderer does not yet apply this model. + + The intrinsic parameters (:attr:`fx`, :attr:`fy`, :attr:`cx`, :attr:`cy`) follow the OpenCV + convention. When a distortion model is present, they take precedence over the focal-length + and aperture projection of :class:`PinholeCameraCfg`. + + This base class is not meant to be used directly. Use one of the concrete models, e.g. + :class:`OpenCvPinholeDistortionCfg` or :class:`OpenCvFisheyeDistortionCfg`. + """ + + model: str = MISSING + """Discriminator selecting the OpenCV distortion model. Set by each concrete sub-config.""" + + fx: float = MISSING + """Focal length along the image x-axis (in pixels).""" + + fy: float = MISSING + """Focal length along the image y-axis (in pixels).""" + + cx: float = MISSING + """Principal point offset along the image x-axis (in pixels).""" + + cy: float = MISSING + """Principal point offset along the image y-axis (in pixels).""" + + image_size: tuple[int, int] = MISSING + """Calibrated image size as ``(width, height)`` (in pixels).""" + + apply_lens_distortion: bool = True + """Whether to apply the distortion coefficients. Defaults to True. + + If False, the distortion coefficients are authored as zero while the intrinsic parameters + (:attr:`fx`, :attr:`fy`, :attr:`cx`, :attr:`cy`) are kept. This produces an undistorted image + that still uses the calibrated OpenCV intrinsics, which is useful for isolating the effect of + the lens distortion. + """ + + +@configclass +class OpenCvPinholeDistortionCfg(OpenCvDistortionCfg): + """OpenCV pinhole lens-distortion model (radial, tangential and thin-prism terms). + + Corresponds to ``OmniLensDistortionOpenCvPinholeAPI`` under the RTX/OVRTX renderer. The full + coefficient set of the OpenCV rational model is exposed; unused coefficients default to zero. + """ + + model: str = "opencvPinhole" + + k1: float = 0.0 + """First radial distortion coefficient. Defaults to 0.0.""" + + k2: float = 0.0 + """Second radial distortion coefficient. Defaults to 0.0.""" + + k3: float = 0.0 + """Third radial distortion coefficient. Defaults to 0.0.""" + + k4: float = 0.0 + """Fourth radial distortion coefficient (rational model). Defaults to 0.0.""" + + k5: float = 0.0 + """Fifth radial distortion coefficient (rational model). Defaults to 0.0.""" + + k6: float = 0.0 + """Sixth radial distortion coefficient (rational model). Defaults to 0.0.""" + + p1: float = 0.0 + """First tangential distortion coefficient. Defaults to 0.0.""" + + p2: float = 0.0 + """Second tangential distortion coefficient. Defaults to 0.0.""" + + s1: float = 0.0 + """First thin-prism distortion coefficient. Defaults to 0.0.""" + + s2: float = 0.0 + """Second thin-prism distortion coefficient. Defaults to 0.0.""" + + s3: float = 0.0 + """Third thin-prism distortion coefficient. Defaults to 0.0.""" + + s4: float = 0.0 + """Fourth thin-prism distortion coefficient. Defaults to 0.0.""" + + +@configclass +class OpenCvFisheyeDistortionCfg(OpenCvDistortionCfg): + """OpenCV fisheye lens-distortion model. + + Corresponds to ``OmniLensDistortionOpenCvFisheyeAPI`` under the RTX/OVRTX renderer. + + See Also: + :class:`FisheyeCameraCfg` for the USD ``fisheyePolynomial`` projection, an alternative fisheye + model authored directly on the camera rather than as an OpenCV ``fx/fy/cx/cy`` calibration. + """ + + model: str = "opencvFisheye" + + k1: float = 0.0 + """First fisheye distortion coefficient. Defaults to 0.0.""" + + k2: float = 0.0 + """Second fisheye distortion coefficient. Defaults to 0.0.""" + + k3: float = 0.0 + """Third fisheye distortion coefficient. Defaults to 0.0.""" + + k4: float = 0.0 + """Fourth fisheye distortion coefficient. Defaults to 0.0.""" + + @configclass class PinholeCameraCfg(SpawnerCfg): """Configuration parameters for a USD camera prim with pinhole camera settings. @@ -30,7 +149,20 @@ class PinholeCameraCfg(SpawnerCfg): """Type of projection to use for the camera. Defaults to "pinhole". Note: - Currently only "pinhole" is supported. + The stock projection is ``"pinhole"``. An OpenCV ``fx/fy/cx/cy`` + distortion-coefficient + intrinsic model can be applied on top via :attr:`distortion` (see + :class:`OpenCvPinholeDistortionCfg` / :class:`OpenCvFisheyeDistortionCfg`), which the + RTX/OVRTX renderer honors natively. + """ + + distortion: OpenCvDistortionCfg | None = None + """OpenCV lens-distortion model to author on the camera. Defaults to None (no distortion). + + When set, the OpenCV intrinsics and distortion coefficients are authored on the camera prim. + Under the RTX/OVRTX renderer they drive the projection natively and, when a real calibration is + used (``fx != fy`` or an off-center principal point), take precedence over the focal-length and + aperture projection. The Newton renderer does not yet apply this model; the camera renders + undistorted there. """ clipping_range: tuple[float, float] = (0.01, 1e6) @@ -168,6 +300,10 @@ class FisheyeCameraCfg(PinholeCameraCfg): function. .. _fish-eye camera: https://en.wikipedia.org/wiki/Fisheye_lens + + See Also: + :class:`OpenCvFisheyeDistortionCfg` for an OpenCV ``fx/fy/cx/cy`` + distortion-coefficient + fisheye calibration applied via the :attr:`~PinholeCameraCfg.distortion` field. """ func: Callable | str = "{DIR}.sensors:spawn_camera" diff --git a/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py new file mode 100644 index 000000000000..0108b8cb429c --- /dev/null +++ b/source/isaaclab/test/sensors/test_camera_opencv_distortion_ovrtx.py @@ -0,0 +1,217 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Validate that the ``ovrtx`` renderer honors the OpenCV lens-distortion camera model natively. + +The grid-textured ground plane is rendered through a camera carrying an OpenCV pinhole calibration. +With the distortion coefficients applied, OVRTX (which reads ``omni:lensdistortion:opencvPinhole:*`` +from the prim) bows the straight grid lines; with the coefficients muted, the same camera renders them +straight. The two renders must therefore differ meaningfully. The reconstructed ``intrinsic_matrices`` +are also checked end-to-end against the authored, non-square, off-center calibration. + +Notes: + * Runs **kit-less**: this test does not call :class:`~isaaclab.app.AppLauncher`. ``ovrtx`` and Isaac + Sim Kit ship conflicting RTX hydra libraries that cannot co-load; see + :func:`isaaclab.app.sim_launcher.launch_simulation`. + * Uses Newton physics because ``ovrtx`` is incompatible with Kit/Isaac Sim. + * The distorted and reference passes each build their own :class:`SimulationContext`; a single + OVRTX render context applies one lens model, so the two passes must not share a process context. +""" + +from __future__ import annotations + +import importlib.util + +import numpy as np +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.rendering] + +_REQUIRED_MODULES = ("isaaclab_ov", "ovrtx", "isaaclab_newton") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] +_SKIP_MISSING_OVRTX = pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", +) + +if not _MISSING_MODULES: + import torch + from isaaclab_newton.physics.mjwarp_manager_cfg import MJWarpSolverCfg + from isaaclab_newton.physics.newton_manager_cfg import NewtonCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + + import isaaclab.sim as sim_utils + from isaaclab.assets import AssetBaseCfg, RigidObjectCfg + from isaaclab.scene import InteractiveScene, InteractiveSceneCfg + from isaaclab.sensors import Camera, CameraCfg + from isaaclab.sim import SimulationCfg + from isaaclab.sim.spawners.sensors.sensors_cfg import ( + OpenCvDistortionCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + ) + from isaaclab.utils.configclass import configclass + from isaaclab.utils.math import create_rotation_matrix_from_view, quat_from_matrix + +SIM_DT = 1.0 / 60.0 +WIDTH, HEIGHT = 640, 480 +WARMUP_STEPS = 4 + +# Example real-world OpenCV pinhole calibration (fx != fy, off-center principal point). +_CALIB = dict(fx=339.26592887, fy=338.82010626, cx=323.55809091, cy=250.27360914) +_COEFFS = dict(k1=0.07702322, k2=-0.13605453, k3=0.05163219, p1=-0.00024938, p2=-0.00175006) +# scale the (mild) real coefficients so the barrel effect is unambiguous in the assertion +_K_SCALE = 15.0 +# OpenCV fisheye (equidistant) coefficients; the base fisheye projection alone differs strongly from pinhole +_FISHEYE_COEFFS = dict(k1=0.1, k2=-0.05, k3=0.0, k4=0.0) + +_CAM_EYE = (0.0, 0.0, 2.5) +_CAM_TARGET = (1.75, 0.0, 0.0) + + +if not _MISSING_MODULES: + + @configclass + class _DistortionSceneCfg(InteractiveSceneCfg): + """The grid-textured ground plane, a dome light and an off-screen anchor body for Newton.""" + + ground = AssetBaseCfg(prim_path="/World/ground", spawn=sim_utils.GroundPlaneCfg()) + dome_light = AssetBaseCfg( + prim_path="/World/DomeLight", + spawn=sim_utils.DomeLightCfg(intensity=2000.0, color=(0.9, 0.9, 0.9)), + ) + anchor = RigidObjectCfg( + prim_path="{ENV_REGEX_NS}/Anchor", + spawn=sim_utils.CuboidCfg( + size=(0.01, 0.01, 0.01), + rigid_props=sim_utils.RigidBodyPropertiesCfg(), + mass_props=sim_utils.MassPropertiesCfg(mass=0.001), + collision_props=sim_utils.CollisionPropertiesCfg(), + physics_material=sim_utils.RigidBodyMaterialCfg(), + visual_material=sim_utils.PreviewSurfaceCfg(diffuse_color=(0.0, 0.0, 0.0)), + ), + init_state=RigidObjectCfg.InitialStateCfg(pos=(0.0, 0.0, -100.0)), + ) + + +def _pinhole_distortion(apply_lens_distortion: bool) -> OpenCvPinholeDistortionCfg: + """Pinhole OpenCV calibration with the (scaled) SO-101 radial/tangential coefficients.""" + return OpenCvPinholeDistortionCfg( + image_size=(WIDTH, HEIGHT), + apply_lens_distortion=apply_lens_distortion, + **_CALIB, + **{name: value * _K_SCALE for name, value in _COEFFS.items()}, + ) + + +def _fisheye_distortion(apply_lens_distortion: bool) -> OpenCvFisheyeDistortionCfg: + """Fisheye OpenCV calibration reusing the SO-101 intrinsics with fisheye coefficients.""" + return OpenCvFisheyeDistortionCfg( + image_size=(WIDTH, HEIGHT), + apply_lens_distortion=apply_lens_distortion, + **_CALIB, + **_FISHEYE_COEFFS, + ) + + +def _render_grid(distortion: OpenCvDistortionCfg, device: str) -> tuple[np.ndarray, np.ndarray]: + """Render the ground-plane grid through an OpenCV-calibrated OVRTX camera; return ``(rgb, K)``.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext( + SimulationCfg(dt=SIM_DT, physics=NewtonCfg(solver_cfg=MJWarpSolverCfg(), num_substeps=1), device=device) + ) + scene = InteractiveScene(_DistortionSceneCfg(num_envs=1, env_spacing=20.0)) + + rot = tuple( + quat_from_matrix( + create_rotation_matrix_from_view(torch.tensor([_CAM_EYE]), torch.tensor([_CAM_TARGET]), up_axis="Z") + )[0].tolist() + ) + camera = Camera( + CameraCfg( + prim_path="/World/envs/env_.*/Camera", + update_period=0.0, + height=HEIGHT, + width=WIDTH, + data_types=["rgb"], + offset=CameraCfg.OffsetCfg(pos=_CAM_EYE, rot=rot, convention="opengl"), + spawn=PinholeCameraCfg(focal_length=13.6, clipping_range=(0.001, 20.0), distortion=distortion), + renderer_cfg=OVRTXRendererCfg(), + ) + ) + try: + sim.reset() + for _ in range(WARMUP_STEPS): + sim.step() + camera.update(SIM_DT, force_recompute=True) + rgb = camera.data.output["rgb"].torch[0].detach().cpu().float().numpy()[..., :3] + intrinsics = camera.data.intrinsic_matrices.torch[0].detach().cpu().numpy() + return rgb, intrinsics + finally: + del camera + del scene + sim.stop() + sim.clear_instance() + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_distortion_changes_ovrtx_render(device): + """OVRTX must render the distorted and zero-coefficient cameras meaningfully differently.""" + distorted, _ = _render_grid(_pinhole_distortion(True), device=device) + reference, _ = _render_grid(_pinhole_distortion(False), device=device) + + assert distorted.shape == (HEIGHT, WIDTH, 3) + # both frames render content (not degenerate) + assert distorted.std() > 1.0 + assert reference.std() > 1.0 + # OVRTX applied the lens distortion: the two renders differ well beyond render noise + mean_abs_diff = np.abs(distorted.astype(np.float32) - reference.astype(np.float32)).mean() + assert mean_abs_diff > 2.0, f"distorted vs reference differ by only {mean_abs_diff:.3f}/255" + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_distortion_intrinsics_match_authored_ovrtx(device): + """The OVRTX camera reports intrinsics matching the authored, non-square, off-center calibration.""" + _rgb, k = _render_grid(_pinhole_distortion(True), device=device) + + assert k[0, 0] == pytest.approx(_CALIB["fx"], abs=1e-2) + assert k[1, 1] == pytest.approx(_CALIB["fy"], abs=1e-2) + assert k[0, 2] == pytest.approx(_CALIB["cx"], abs=1e-2) + assert k[1, 2] == pytest.approx(_CALIB["cy"], abs=1e-2) + # not the stock fx == fy / centered-principal-point collapse + assert k[0, 0] != k[1, 1] + assert k[0, 2] != pytest.approx(WIDTH / 2) + + +@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.isaacsim_ci +@_SKIP_MISSING_OVRTX +def test_opencv_fisheye_distortion_renders_through_ovrtx(device): + """OVRTX honors the OpenCV fisheye schema: its render differs meaningfully from the pinhole projection. + + The same calibrated camera is rendered under the OpenCV fisheye model and under an undistorted + pinhole. The fisheye equidistant projection bends the straight grid, so the two frames must differ + well beyond render noise, and the reported intrinsics must still match the authored calibration. + """ + fisheye, k = _render_grid(_fisheye_distortion(True), device=device) + pinhole, _ = _render_grid(_pinhole_distortion(False), device=device) + + assert fisheye.shape == (HEIGHT, WIDTH, 3) + # both frames render content (not degenerate) + assert fisheye.std() > 1.0 + assert pinhole.std() > 1.0 + # OVRTX applied the fisheye projection: it differs from the pinhole render well beyond render noise + mean_abs_diff = np.abs(fisheye.astype(np.float32) - pinhole.astype(np.float32)).mean() + assert mean_abs_diff > 2.0, f"fisheye vs pinhole differ by only {mean_abs_diff:.3f}/255" + # the fisheye camera still reports the authored, non-square, off-center calibration + assert k[0, 0] == pytest.approx(_CALIB["fx"], abs=1e-2) + assert k[1, 1] == pytest.approx(_CALIB["fy"], abs=1e-2) + assert k[0, 2] == pytest.approx(_CALIB["cx"], abs=1e-2) + assert k[1, 2] == pytest.approx(_CALIB["cy"], abs=1e-2) diff --git a/source/isaaclab/test/sensors/test_opencv_distortion.py b/source/isaaclab/test/sensors/test_opencv_distortion.py new file mode 100644 index 000000000000..832eaf956390 --- /dev/null +++ b/source/isaaclab/test/sensors/test_opencv_distortion.py @@ -0,0 +1,344 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Kit-less tests for the renderer-agnostic OpenCV lens-distortion camera model. + +Two renderer-independent code paths are covered without a running simulation app, renderer or GPU: + +* Authoring: :func:`isaaclab.sim.spawners.sensors.spawn_camera` with a ``distortion`` cfg authors the + ``omni:lensdistortion:*`` USD API on the camera prim (which the RTX/OVRTX renderer round-trips + through the USD export it loads). +* Readback: :meth:`isaaclab.sensors.camera.Camera._update_intrinsic_matrices` reconstructs + ``camera.data.intrinsic_matrices`` from the authored ``fx/fy/cx/cy`` (which may be non-square or + off-center) instead of assuming ``fx == fy`` and a centered principal point. + +The renderer actually rendering *through* the distortion is a backend-specific concern covered in +``test_camera_opencv_distortion_ovrtx.py``. +""" + +from __future__ import annotations + +import importlib.util +import logging +from types import SimpleNamespace + +import numpy as np +import pytest + +_REQUIRED_MODULES = ("isaaclab", "pxr", "warp") +_MISSING_MODULES = [module for module in _REQUIRED_MODULES if importlib.util.find_spec(module) is None] + +pytestmark = [ + pytest.mark.unit, + pytest.mark.skipif( + bool(_MISSING_MODULES), + reason=f"requires optional modules: {', '.join(_MISSING_MODULES)}", + ), +] + +if not _MISSING_MODULES: + import torch + + from pxr import Gf, Sdf, Usd, UsdGeom + + import isaaclab.sim as sim_utils + from isaaclab.sensors.camera.camera import Camera + from isaaclab.sim.spawners.sensors.sensors import spawn_camera + from isaaclab.sim.spawners.sensors.sensors_cfg import ( + FisheyeCameraCfg, + OpenCvFisheyeDistortionCfg, + OpenCvPinholeDistortionCfg, + PinholeCameraCfg, + ) + + +# Real SO-101 wrist-camera calibration: exercises fx != fy and an off-center principal point, which +# stock Isaac Lab camera cfgs cannot express. +_PINHOLE_CALIB = dict( + fx=339.26592887, + fy=338.82010626, + cx=323.55809091, + cy=250.27360914, + image_size=(640, 480), + k1=0.07702322, + k2=-0.13605453, + k3=0.05163219, + p1=-0.00024938, + p2=-0.00175006, +) + + +""" +Authoring: spawn_camera with a distortion cfg. +""" + + +def _spawn_camera_on_new_stage(cfg, prim_path="/World/envs/env_0/Camera"): + """Create a fresh in-memory stage and spawn a camera prim carrying ``cfg`` on it.""" + stage = sim_utils.create_new_stage() + UsdGeom.Xform.Define(stage, "/World") + UsdGeom.Xform.Define(stage, "/World/envs") + UsdGeom.Xform.Define(stage, "/World/envs/env_0") + # spawn_camera is @clone-decorated; call the undecorated function to spawn a single prim. + prim = spawn_camera.__wrapped__(prim_path, cfg) + return stage, prim + + +def test_pinhole_distortion_authors_opencv_api(): + """Spawning a pinhole camera with a distortion cfg authors the OpenCV pinhole USD API.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=True, **_PINHOLE_CALIB)) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvPinhole" + assert prim.GetAttribute("omni:lensdistortion:model").Get() == "opencvPinhole" + assert "OmniLensDistortionOpenCvPinholeAPI" in prim.GetMetadata("apiSchemas").GetAppliedItems() + # intrinsics + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(_PINHOLE_CALIB["fx"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:fy").Get() == pytest.approx(_PINHOLE_CALIB["fy"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:cx").Get() == pytest.approx(_PINHOLE_CALIB["cx"], abs=1e-3) + assert prim.GetAttribute(f"{prefix}:cy").Get() == pytest.approx(_PINHOLE_CALIB["cy"], abs=1e-3) + assert tuple(prim.GetAttribute(f"{prefix}:imageSize").Get()) == (640, 480) + # distortion coefficients (applied) + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(_PINHOLE_CALIB["k1"], abs=1e-6) + assert prim.GetAttribute(f"{prefix}:p2").Get() == pytest.approx(_PINHOLE_CALIB["p2"], abs=1e-6) + # unused higher-order coefficients default to zero + assert prim.GetAttribute(f"{prefix}:k6").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:s4").Get() == pytest.approx(0.0) + + +def test_apply_lens_distortion_false_zeros_coefficients(): + """With ``apply_lens_distortion=False`` the intrinsics are kept but coefficients are muted to zero.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=False, **_PINHOLE_CALIB)) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvPinhole" + # intrinsics are still authored + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(_PINHOLE_CALIB["fx"], abs=1e-3) + # coefficients are zeroed despite non-zero cfg values + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:k2").Get() == pytest.approx(0.0) + assert prim.GetAttribute(f"{prefix}:p1").Get() == pytest.approx(0.0) + + +def test_fisheye_distortion_authors_opencv_fisheye_api(): + """A distortion cfg with the fisheye model authors the OpenCV fisheye USD API.""" + cfg = FisheyeCameraCfg( + distortion=OpenCvFisheyeDistortionCfg( + fx=300.0, fy=300.0, cx=320.0, cy=240.0, image_size=(640, 480), k1=0.1, k2=0.02, k3=0.0, k4=0.0 + ) + ) + _stage, prim = _spawn_camera_on_new_stage(cfg) + + prefix = "omni:lensdistortion:opencvFisheye" + assert prim.GetAttribute("omni:lensdistortion:model").Get() == "opencvFisheye" + assert "OmniLensDistortionOpenCvFisheyeAPI" in prim.GetMetadata("apiSchemas").GetAppliedItems() + assert prim.GetAttribute(f"{prefix}:fx").Get() == pytest.approx(300.0) + assert prim.GetAttribute(f"{prefix}:k1").Get() == pytest.approx(0.1) + assert prim.GetAttribute(f"{prefix}:k4").Get() == pytest.approx(0.0) + + +def test_distortion_attributes_survive_usd_export(): + """The authored OpenCV distortion API survives serialization, which the RTX/OVRTX path consumes.""" + cfg = PinholeCameraCfg(distortion=OpenCvPinholeDistortionCfg(apply_lens_distortion=True, **_PINHOLE_CALIB)) + stage, _prim = _spawn_camera_on_new_stage(cfg) + + exported = stage.ExportToString() + assert "OmniLensDistortionOpenCvPinholeAPI" in exported + assert "omni:lensdistortion:opencvPinhole:fx" in exported + assert "omni:lensdistortion:model" in exported + + +def test_camera_without_distortion_authors_no_opencv_api(): + """A camera cfg without a distortion model does not author any ``omni:lensdistortion`` attributes.""" + cfg = PinholeCameraCfg() + _stage, prim = _spawn_camera_on_new_stage(cfg) + + assert not prim.GetAttribute("omni:lensdistortion:model").IsValid() + applied = prim.GetMetadata("apiSchemas") + applied_items = list(applied.GetAppliedItems()) if applied else [] + assert not any("LensDistortion" in item for item in applied_items) + + +""" +Readback: Camera._update_intrinsic_matrices with an authored distortion model. +""" + + +def _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width=640, height=480): + """Build an in-memory ``UsdGeom.Camera`` carrying an OpenCV pinhole distortion model.""" + stage = Usd.Stage.CreateInMemory() + cam = UsdGeom.Camera.Define(stage, "/Camera") + prim = cam.GetPrim() + prim.CreateAttribute("omni:lensdistortion:model", Sdf.ValueTypeNames.Token).Set("opencvPinhole") + prefix = "omni:lensdistortion:opencvPinhole" + for name, value in (("fx", fx), ("fy", fy), ("cx", cx), ("cy", cy)): + prim.CreateAttribute(f"{prefix}:{name}", Sdf.ValueTypeNames.Float).Set(float(value)) + prim.CreateAttribute(f"{prefix}:imageSize", Sdf.ValueTypeNames.Int2).Set(Gf.Vec2i(width, height)) + return stage, cam + + +def _read_back_intrinsics(cam, width, height): + """Invoke :meth:`Camera._update_intrinsic_matrices` on a minimal fake camera and return K.""" + captured = {} + + def _capture_state(env_ids=None, intrinsics_src=None, update_intrinsics=False): + captured["K"] = intrinsics_src.numpy() + + fake = Camera.__new__(Camera) + fake._sensor_prims = [cam] + fake._device = "cpu" + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False + fake.cfg = SimpleNamespace(height=height, width=width) + fake._resolve_env_ids_np = lambda env_ids: np.array([0]) + fake._update_camera_state = _capture_state + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + Camera._update_intrinsic_matrices(fake) + return captured["K"][0] + + +def test_readback_uses_authored_fx_fy_cx_cy(): + """The reconstructed intrinsic matrix reflects the authored, non-square, off-center calibration.""" + fx, fy, cx, cy = 339.26592887, 338.82010626, 323.55809091, 250.27360914 + width, height = 640, 480 + _stage, cam = _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width, height) + + k = _read_back_intrinsics(cam, width, height) + + assert k[0, 0] == pytest.approx(fx, abs=1e-2) + assert k[1, 1] == pytest.approx(fy, abs=1e-2) + assert k[0, 2] == pytest.approx(cx, abs=1e-2) + assert k[1, 2] == pytest.approx(cy, abs=1e-2) + assert k[2, 2] == pytest.approx(1.0) + + +def test_readback_preserves_non_square_and_off_center(): + """The readback must not collapse to the stock ``fx == fy`` / centered principal point.""" + fx, fy, cx, cy = 339.26592887, 338.82010626, 323.55809091, 250.27360914 + width, height = 640, 480 + _stage, cam = _camera_prim_with_pinhole_distortion(fx, fy, cx, cy, width, height) + + k = _read_back_intrinsics(cam, width, height) + + # non-square pixels are preserved (the old readback forced fx == fy) + assert k[0, 0] != k[1, 1] + # principal point is off-center (the old readback forced width/2, height/2) + assert k[0, 2] != pytest.approx(width / 2) + assert k[1, 2] != pytest.approx(height / 2) + + +def _camera_prim_with_model_token_only(): + """Build a camera prim that declares the distortion model token but authors no fx/fy/cx/cy.""" + stage = Usd.Stage.CreateInMemory() + cam = UsdGeom.Camera.Define(stage, "/Camera") + cam.GetPrim().CreateAttribute("omni:lensdistortion:model", Sdf.ValueTypeNames.Token).Set("opencvPinhole") + return stage, cam + + +def test_readback_missing_intrinsics_falls_back_to_focal_length(): + """A model token without fx/fy/cx/cy falls back to the focal-length projection instead of raising.""" + width, height = 640, 480 + _stage, cam = _camera_prim_with_model_token_only() + + k = _read_back_intrinsics(cam, width, height) + + # focal-length/aperture projection: square pixels, centered principal point + assert k[0, 0] == pytest.approx(k[1, 1]) + assert k[0, 2] == pytest.approx(width / 2) + assert k[1, 2] == pytest.approx(height / 2) + assert k[2, 2] == pytest.approx(1.0) + + +def test_readback_distinct_image_size_mismatches_each_warn(): + """Distinct authored image sizes across a camera's prims each warn, not only the first.""" + width, height = 320, 240 + _stage_a, cam_a = _camera_prim_with_pinhole_distortion(300.0, 300.0, 160.0, 120.0, 640, 480) + _stage_b, cam_b = _camera_prim_with_pinhole_distortion(300.0, 300.0, 160.0, 120.0, 1280, 720) + + fake = Camera.__new__(Camera) + fake._sensor_prims = [cam_a, cam_b] + fake._device = "cpu" + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False + fake.cfg = SimpleNamespace(height=height, width=width) + fake._resolve_env_ids_np = lambda env_ids: np.array([0, 1]) + fake._update_camera_state = lambda **kwargs: None + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + Camera._update_intrinsic_matrices(fake) + + assert fake._warned_distortion_image_sizes == {(640, 480), (1280, 720)} + + +def test_set_intrinsic_matrices_skips_only_distortion_cameras_in_batch(): + """In a mixed batch only the distortion camera is skipped (with a warning); a plain camera is updated. + + The authored ``omni:lensdistortion:*`` fx/fy/cx/cy are the readback's source of truth, so the + focal-length/aperture write would be discarded for a distortion camera. Skipping the whole call would + also drop ordinary selected cameras; only the distortion entries must be left untouched. + """ + width, height = 640, 480 + _stage_d, distortion_cam = _camera_prim_with_pinhole_distortion(339.0, 338.0, 323.0, 250.0, width, height) + plain_stage = Usd.Stage.CreateInMemory() + plain_cam = UsdGeom.Camera.Define(plain_stage, "/PlainCamera") + + captured = {} + + def _capture_state(env_ids=None, intrinsics_src=None, update_intrinsics=False): + captured["K"] = intrinsics_src.numpy() + + fake = Camera.__new__(Camera) + fake._sensor_prims = [distortion_cam, plain_cam] + fake._device = "cpu" + fake.cfg = SimpleNamespace(height=height, width=width) + fake._warned_distortion_image_sizes = set() + fake._warned_distortion_missing_intrinsics = False + fake._resolve_env_ids_np = lambda env_ids: np.array([0, 1]) + fake._update_camera_state = _capture_state + # attributes touched by ``__del__``/``_clear_callbacks`` when the fake object is garbage collected + fake._initialize_handle = None + fake._invalidate_initialize_handle = None + fake._prim_deletion_handle = None + fake._debug_vis_handle = None + fake._renderer = None + fake._render_data = None + + messages: list[str] = [] + handler = logging.Handler() + handler.emit = lambda record: messages.append(record.getMessage()) + cam_logger = logging.getLogger("isaaclab.sensors.camera.camera") + cam_logger.addHandler(handler) + # row 0 targets the distortion camera (skipped); row 1 recalibrates the plain camera to fx = fy = 500 + requested = torch.tensor( + [ + [[999.0, 0.0, 111.0], [0.0, 999.0, 222.0], [0.0, 0.0, 1.0]], + [[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]], + ], + dtype=torch.float32, + ) + try: + Camera.set_intrinsic_matrices(fake, requested, env_ids=[0, 1]) + finally: + cam_logger.removeHandler(handler) + + k = captured["K"] + # the distortion camera keeps its authored calibration (skipped, request ignored) + assert k[0, 0, 0] == pytest.approx(339.0, abs=1e-2) + # the plain camera reflects the requested focal length (updated, not over-skipped) + assert k[1, 0, 0] == pytest.approx(500.0, abs=1e-2) + assert any("skipped" in message.lower() for message in messages) diff --git a/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst b/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst new file mode 100644 index 000000000000..c19be2eddb53 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/opencv-lens-distortion-cameras.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Added a warning when a camera cfg carries an OpenCV lens-distortion model + (``spawn.distortion``) under the Newton renderer, which does not yet apply the model; the camera + renders undistorted. The distortion cfg is renderer-agnostic and remains a documented extension + point for a future Newton implementation. diff --git a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py index 089be5ffc313..72d02c3ca6e8 100644 --- a/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py +++ b/source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py @@ -384,6 +384,18 @@ def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: owns the sentinel-resolution + cfg-normalization step. Newton has no USD-side overrides to author beyond this. """ + # NOTE: OpenCV lens distortion (``spawn.distortion``) is not yet applied by the Newton + # renderer. The distortion cfg is renderer-agnostic and could be piped through Newton's warp + # ray-tracing utilities here in the future; for now the camera renders undistorted. This is + # the intended extension point. + spawn = getattr(spec.cfg, "spawn", None) + if getattr(spawn, "distortion", None) is not None: + logger.warning( + "OpenCV lens distortion is set on the camera cfg but is not yet applied by the Newton" + " renderer: it derives a single field of view from fy, so the distortion coefficients," + " the principal point, and a non-square fx are ignored and the camera renders as a" + " centered, square-pixel pinhole. Use the RTX/OVRTX renderer to apply the full model." + ) if spec.cfg.isp_cfg is None: return try: