diff --git a/CodenameOne/src/com/codename1/ar/AR.java b/CodenameOne/src/com/codename1/ar/AR.java new file mode 100644 index 00000000000..1305487e3be --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/AR.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.impl.ARImpl; +import com.codename1.io.Log; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +import java.io.IOException; + +/// Entry point for the cross-platform augmented reality API. +/// +/// AR sessions track the device's pose in the real world through the camera, +/// detect surfaces, and composite virtual 3D content into the camera image. +/// The typical flow: check `#isSupported()`, `#open(ARSessionOptions)` a +/// session, add its `ARSession#createView()` to a form, then place content by +/// hit testing taps against detected planes and hanging `ARNode`s on the +/// resulting anchors. +/// +/// **Permissions and dependencies**: simply referencing classes in this +/// package causes the build pipeline to inject the camera permission, the +/// `NSCameraUsageDescription` plist entry (iOS, override via the +/// `ios.NSCameraUsageDescription` build hint) and the ARCore dependency +/// (Android). Devices without AR support report `#isSupported()` false; +/// always guard AR functionality behind that check. +/// +/// ```java +/// if (AR.isSupported()) { +/// ARSession s = AR.open(new ARSessionOptions()); +/// Form f = new Form("AR", new BorderLayout()); +/// f.add(BorderLayout.CENTER, s.createView()); +/// f.show(); +/// // on tap: hit test and place a model +/// s.hitTest(xNorm, yNorm).ready(hits -> { +/// if (hits.length > 0) { +/// ARAnchor a = hits[0].createAnchor(); +/// a.setNode(new ARNode(ARModel.fromGltf(modelBytes))); +/// } +/// }); +/// } +/// ``` +public final class AR { + private static final Object ACTIVE_LOCK = new Object(); + private static ARSession active; + + private AR() { + } + + /// True when the running platform has a working AR implementation. False + /// on platforms and devices without AR support. + public static boolean isSupported() { + return Display.getInstance().getARBackend() != null; + } + + /// Returns which AR features this device supports. On unsupported + /// platforms every capability reads false. + public static ARCapabilities getCapabilities() { + ARImpl probe = newImpl(); + if (probe == null) { + return ARCapabilities.UNSUPPORTED; + } + try { + ARCapabilities caps = probe.getCapabilities(); + return caps == null ? ARCapabilities.UNSUPPORTED : caps; + } finally { + closeQuietly(probe); + } + } + + /// Opens an AR session. Throws `IllegalStateException` when AR is + /// unsupported or a session is already open; close the old session first. + /// + /// #### Parameters + /// + /// - `opts`: the session configuration; null uses the defaults + /// + /// #### Returns + /// + /// the running session + public static ARSession open(ARSessionOptions opts) { + if (opts == null) { + opts = new ARSessionOptions(); + } + // The check-and-set is atomic so the "one open session at a time" + // contract holds under concurrent open() calls; contention is + // essentially zero since opening AR is a foreground user action. + synchronized (ACTIVE_LOCK) { + if (active != null && !active.isClosed()) { + throw new IllegalStateException( + "Only one ARSession may be open at a time. Close the existing session first."); + } + ARImpl impl = newImpl(); + if (impl == null) { + throw new IllegalStateException("AR is not supported on this platform."); + } + ARSession session = new ARSession(impl, opts); + try { + impl.open(opts); + } catch (IOException e) { + session.close(); + throw new RuntimeException("Could not open the AR session", e); + } + active = session; + return session; + } + } + + /// Requests the camera permission needed for AR. The callback receives + /// true when granted; it is invoked on the EDT. + /// + /// #### Parameters + /// + /// - `callback`: receives the grant result; may be null + public static void requestPermissions(final SuccessCallback callback) { + final ARImpl impl = newImpl(); + if (impl == null) { + fireLater(callback, Boolean.FALSE); + return; + } + AsyncResource result = new AsyncResource(); + result.ready(new SuccessCallback() { + @Override public void onSucess(Boolean value) { + closeQuietly(impl); + if (callback != null) { + callback.onSucess(Boolean.valueOf(value != null && value.booleanValue())); + } + } + }); + result.except(new SuccessCallback() { + @Override public void onSucess(Throwable t) { + closeQuietly(impl); + if (callback != null) { + callback.onSucess(Boolean.FALSE); + } + } + }); + impl.requestPermissions(result); + } + + private static void fireLater(final SuccessCallback callback, final Boolean value) { + if (callback == null) { + return; + } + Display.getInstance().callSerially(new Runnable() { + @Override public void run() { + callback.onSucess(value); + } + }); + } + + private static void closeQuietly(ARImpl impl) { + try { + impl.close(); + } catch (Throwable t) { + Log.e(t); + } + } + + private static ARImpl newImpl() { + return Display.getInstance().getARBackend(); + } + + // Identity comparison is intentional: only the exact ARSession instance + // returned from open() may clear the active slot. + @SuppressWarnings("PMD.CompareObjectsWithEquals") + static void clearActive(ARSession s) { + synchronized (ACTIVE_LOCK) { + if (active == s) { + active = null; + } + } + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARAnchor.java b/CodenameOne/src/com/codename1/ar/ARAnchor.java new file mode 100644 index 00000000000..5c4592323f7 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARAnchor.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// A fixed position and orientation in the real world that the AR session +/// keeps tracking as its understanding of the environment improves. Anchors +/// are the attachment points for virtual content: create one with +/// `ARSession#createAnchor(ARPose)` or `ARHitResult#createAnchor()` and hang +/// an `ARNode` on it with `#setNode(ARNode)`. +/// +/// The session refines anchor poses over time; updates are delivered through +/// the session's `ARAnchorListener` and reflected by `#getPose()`. +public class ARAnchor { + private final String id; + private ARPose pose; + private ARTrackingState trackingState = ARTrackingState.TRACKING; + private ARNode node; + private boolean detached; + ARSession session; + + /// Creates an anchor. Intended for platform implementations and tests; + /// applications create anchors through the session. + /// + /// #### Parameters + /// + /// - `id`: the stable identifier of this anchor + /// + /// - `pose`: the initial world pose + public ARAnchor(String id, ARPose pose) { + if (id == null || id.length() == 0) { + throw new IllegalArgumentException("id is required"); + } + this.id = id; + this.pose = pose == null ? ARPose.IDENTITY : pose; + } + + /// The stable identifier of this anchor. + public String getId() { + return id; + } + + /// The current world pose. Updated on the EDT as tracking refines. + public ARPose getPose() { + return pose; + } + + /// The tracking quality of this anchor. + public ARTrackingState getTrackingState() { + return trackingState; + } + + /// The content root rendered at this anchor, or null when none is + /// attached. + public ARNode getNode() { + return node; + } + + /// Attaches (or replaces) the content root rendered at this anchor. Pass + /// null to remove the content while keeping the anchor. + /// + /// #### Parameters + /// + /// - `node`: the root node to render at this anchor, or null + // Identity comparison is intentional: re-attaching the exact same node + // instance must not clear its session wiring first. + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public void setNode(ARNode node) { + if (detached) { + throw new IllegalStateException("anchor is detached"); + } + if (node != null && node.getParent() != null) { + throw new IllegalArgumentException("the anchor content root may not have a parent node"); + } + ARNode old = this.node; + if (old != null && old != node) { + old.session = null; + old.anchorId = null; + } + this.node = node; + if (node != null) { + node.session = session; + node.anchorId = id; + } + if (session != null) { + session.anchorNodeChangedInternal(this, node); + } + } + + /// Removes this anchor (and any attached content) from the session. + /// No-op when already detached. + public void detach() { + if (detached) { + return; + } + detached = true; + if (node != null) { + node.session = null; + node.anchorId = null; + } + if (session != null) { + session.anchorDetachedInternal(this); + } + } + + /// True once `#detach()` has been called or the platform removed the + /// anchor. + public boolean isDetached() { + return detached; + } + + /// Updates the pose and tracking state from the platform. Called by the + /// session on the EDT. + void update(ARPose pose, ARTrackingState state) { + if (pose != null) { + this.pose = pose; + } + if (state != null) { + this.trackingState = state; + } + } + + /// Marks the anchor detached without calling back into the session. Used + /// when the platform removes the anchor. + void markDetached() { + detached = true; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARAnchorEvent.java b/CodenameOne/src/com/codename1/ar/ARAnchorEvent.java new file mode 100644 index 00000000000..a15c966ad6e --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARAnchorEvent.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Describes a change to an `ARAnchor`, delivered to `ARAnchorListener`s on +/// the EDT. Anchors recognized by the platform - detected reference images and +/// tracked faces - arrive here as `ARImageAnchor` / `ARFaceAnchor` subtypes; +/// use `instanceof` to handle them specifically. +public final class ARAnchorEvent { + /// The kind of change that occurred. + public enum Kind { + /// The anchor was added - either created by the application or + /// recognized by the platform (image or face). + ADDED, + + /// The anchor's pose or tracking state was refined. + UPDATED, + + /// The anchor was removed and is now detached. + REMOVED + } + + private final Kind kind; + private final ARAnchor anchor; + private final ARSession session; + + ARAnchorEvent(Kind kind, ARAnchor anchor, ARSession session) { + this.kind = kind; + this.anchor = anchor; + this.session = session; + } + + /// The kind of change that occurred. + public Kind getKind() { + return kind; + } + + /// The anchor that changed. + public ARAnchor getAnchor() { + return anchor; + } + + /// The session the anchor belongs to. + public ARSession getSession() { + return session; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARAnchorListener.java b/CodenameOne/src/com/codename1/ar/ARAnchorListener.java new file mode 100644 index 00000000000..9da53e476cc --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARAnchorListener.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Observes anchor changes, including platform-recognized images and faces. +/// Register through `ARSession#addAnchorListener(ARAnchorListener)`; events +/// are delivered on the EDT. +public interface ARAnchorListener { + /// Invoked on the EDT when an anchor is added, refined or removed. + /// + /// #### Parameters + /// + /// - `event`: the change description + void anchorChanged(ARAnchorEvent event); +} diff --git a/CodenameOne/src/com/codename1/ar/ARCapabilities.java b/CodenameOne/src/com/codename1/ar/ARCapabilities.java new file mode 100644 index 00000000000..dcaa197854c --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARCapabilities.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Describes which AR features the current device supports. Individual +/// features are hardware-gated on both platforms - for example face tracking +/// needs a capable front camera - so check the specific capability before +/// opening a session in that mode. Obtain through `AR#getCapabilities()`. +public final class ARCapabilities { + /// A capabilities instance with every feature unsupported, returned on + /// platforms without an AR backend. + public static final ARCapabilities UNSUPPORTED = + new ARCapabilities(false, false, false, false, false); + + private final boolean worldTracking; + private final boolean planeDetection; + private final boolean imageTracking; + private final boolean faceTracking; + private final boolean lightEstimation; + + /// Creates a capabilities descriptor. Intended for platform + /// implementations and tests; applications obtain capabilities through + /// `AR#getCapabilities()`. + public ARCapabilities(boolean worldTracking, boolean planeDetection, + boolean imageTracking, boolean faceTracking, + boolean lightEstimation) { + this.worldTracking = worldTracking; + this.planeDetection = planeDetection; + this.imageTracking = imageTracking; + this.faceTracking = faceTracking; + this.lightEstimation = lightEstimation; + } + + /// True when world tracking (`ARTrackingMode#WORLD`) is supported. + public boolean isWorldTrackingSupported() { + return worldTracking; + } + + /// True when plane detection is supported in world tracking sessions. + public boolean isPlaneDetectionSupported() { + return planeDetection; + } + + /// True when reference image detection is supported. + public boolean isImageTrackingSupported() { + return imageTracking; + } + + /// True when face tracking (`ARTrackingMode#FACE`) is supported on this + /// device. + public boolean isFaceTrackingSupported() { + return faceTracking; + } + + /// True when the session can estimate real-world lighting. + public boolean isLightEstimationSupported() { + return lightEstimation; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARFaceAnchor.java b/CodenameOne/src/com/codename1/ar/ARFaceAnchor.java new file mode 100644 index 00000000000..10c72af3404 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARFaceAnchor.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// An anchor tracking a face in a `ARTrackingMode#FACE` session. The anchor +/// pose is centered on the head; named regions and an optional face mesh give +/// finer geometry for effects such as filters or virtual try-on. +/// +/// Platform capabilities differ: region poses and the mesh may be null on +/// platforms that do not supply them - always handle the null case. Delivered +/// through the session's `ARAnchorListener`; use `instanceof` to distinguish +/// face anchors from plain anchors. +public class ARFaceAnchor extends ARAnchor { + private ARPose[] regionPoses; + private float[] meshVertices; + private int[] meshTriangles; + + /// Creates a face anchor. Intended for platform implementations and + /// tests. + /// + /// #### Parameters + /// + /// - `id`: the stable identifier of this anchor + /// + /// - `pose`: the initial world pose, centered on the head + public ARFaceAnchor(String id, ARPose pose) { + super(id, pose); + } + + /// The pose of a named face region in world space, or null when the + /// platform does not supply that region. + /// + /// #### Parameters + /// + /// - `region`: the face region to query + /// + /// #### Returns + /// + /// the region pose or null + public ARPose getRegionPose(ARFaceRegion region) { + if (region == null || regionPoses == null) { + return null; + } + int i = region.ordinal(); + return i < regionPoses.length ? regionPoses[i] : null; + } + + /// The face mesh vertices as `x, y, z` triples in the anchor's local + /// frame, or null when the platform supplies no mesh. The array is the + /// live buffer updated each frame; copy it if you need a stable snapshot. + public float[] getMeshVertices() { + return meshVertices; + } + + /// The face mesh triangle indices (three per triangle into + /// `#getMeshVertices()`), or null when the platform supplies no mesh. + public int[] getMeshTriangles() { + return meshTriangles; + } + + /// Updates the face geometry from the platform. Called by the session on + /// the EDT. + void updateFace(ARPose[] regionPoses, float[] meshVertices, int[] meshTriangles) { + if (regionPoses != null) { + this.regionPoses = regionPoses; + } + if (meshVertices != null) { + this.meshVertices = meshVertices; + } + if (meshTriangles != null) { + this.meshTriangles = meshTriangles; + } + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARFaceRegion.java b/CodenameOne/src/com/codename1/ar/ARFaceRegion.java new file mode 100644 index 00000000000..6e97bdf31e5 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARFaceRegion.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Named face regions whose poses can be queried from an `ARFaceAnchor`. +/// Platform availability differs: ARCore natively supplies the nose tip and +/// forehead regions, while eye poses are an ARKit capability. Query +/// `ARFaceAnchor#getRegionPose(ARFaceRegion)` and handle a null result. +public enum ARFaceRegion { + /// The tip of the nose. + NOSE_TIP, + + /// The left side of the forehead (the person's left). + FOREHEAD_LEFT, + + /// The right side of the forehead (the person's right). + FOREHEAD_RIGHT, + + /// The center of the left eye (the person's left). May be unavailable on + /// some platforms. + LEFT_EYE, + + /// The center of the right eye (the person's right). May be unavailable on + /// some platforms. + RIGHT_EYE +} diff --git a/CodenameOne/src/com/codename1/ar/ARHitResult.java b/CodenameOne/src/com/codename1/ar/ARHitResult.java new file mode 100644 index 00000000000..d9b3b9df29b --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARHitResult.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// One intersection returned by `ARSession#hitTest(float, float)`: the world +/// pose where the ray from the screen point met real-world geometry. Call +/// `#createAnchor()` to place content at the hit. +public final class ARHitResult { + /// The kind of real-world geometry the hit ray intersected. + public enum Type { + /// A detected `ARPlane`. + PLANE, + + /// A surface the platform estimates exists but has not fully detected + /// as a plane yet. + ESTIMATED_PLANE, + + /// An individual tracked feature point. + FEATURE_POINT + } + + private final ARPose pose; + private final float distance; + private final Type type; + private final ARPlane plane; + private final Object nativeHandle; + ARSession session; + + /// Creates a hit result. Intended for platform implementations and tests; + /// applications receive hit results from `ARSession#hitTest(float, float)`. + /// + /// #### Parameters + /// + /// - `pose`: the world pose of the intersection, local Y along the + /// surface normal + /// + /// - `distance`: the distance from the camera in meters + /// + /// - `type`: the kind of geometry hit + /// + /// - `plane`: the plane that was hit, or null for non-plane hits + /// + /// - `nativeHandle`: an opaque platform token that lets + /// `#createAnchor()` anchor to the exact native hit, or null + public ARHitResult(ARPose pose, float distance, Type type, ARPlane plane, + Object nativeHandle) { + if (pose == null) { + throw new IllegalArgumentException("pose is required"); + } + if (type == null) { + throw new IllegalArgumentException("type is required"); + } + this.pose = pose; + this.distance = distance; + this.type = type; + this.plane = plane; + this.nativeHandle = nativeHandle; + } + + /// The world pose of the intersection. Local Y points along the surface + /// normal. + public ARPose getPose() { + return pose; + } + + /// The distance from the camera to the hit in meters. + public float getDistance() { + return distance; + } + + /// The kind of geometry the ray intersected. + public Type getType() { + return type; + } + + /// The plane that was hit, or null for non-plane hits. + public ARPlane getPlane() { + return plane; + } + + /// The opaque platform token backing this hit. Intended for platform + /// implementations. + public Object getNativeHandle() { + return nativeHandle; + } + + /// Creates an anchor at this hit, letting the platform anchor to the + /// exact native raycast result when available. + /// + /// #### Returns + /// + /// the new anchor, registered with the session + public ARAnchor createAnchor() { + if (session == null) { + throw new IllegalStateException( + "this hit result is not attached to a session"); + } + return session.createAnchorFromHitInternal(this); + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARImageAnchor.java b/CodenameOne/src/com/codename1/ar/ARImageAnchor.java new file mode 100644 index 00000000000..bc701705940 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARImageAnchor.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// An anchor tracking a recognized `ARReferenceImage` in the real world. The +/// anchor pose is centered on the physical image with local X/Z spanning the +/// image surface. Delivered through the session's `ARAnchorListener`; use +/// `instanceof` to distinguish image anchors from plain anchors. +public class ARImageAnchor extends ARAnchor { + private final String referenceImageName; + private final float estimatedPhysicalWidth; + + /// Creates an image anchor. Intended for platform implementations and + /// tests. + /// + /// #### Parameters + /// + /// - `id`: the stable identifier of this anchor + /// + /// - `pose`: the initial world pose, centered on the physical image + /// + /// - `referenceImageName`: the `ARReferenceImage#getName()` of the + /// recognized image + /// + /// - `estimatedPhysicalWidth`: the platform's estimate of the physical + /// image width in meters + public ARImageAnchor(String id, ARPose pose, String referenceImageName, + float estimatedPhysicalWidth) { + super(id, pose); + this.referenceImageName = referenceImageName; + this.estimatedPhysicalWidth = estimatedPhysicalWidth; + } + + /// The name of the recognized reference image. + public String getReferenceImageName() { + return referenceImageName; + } + + /// The platform's estimate of the physical image width in meters. May + /// differ slightly from the registered + /// `ARReferenceImage#getPhysicalWidthMeters()`. + public float getEstimatedPhysicalWidth() { + return estimatedPhysicalWidth; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARLightEstimate.java b/CodenameOne/src/com/codename1/ar/ARLightEstimate.java new file mode 100644 index 00000000000..246066907fe --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARLightEstimate.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// An estimate of the real-world lighting around the device, used to shade +/// virtual content so it blends with the camera image. Poll it through +/// `ARSession#getLightEstimate()`; light changes every frame, so no events are +/// fired for it. +/// +/// The ambient intensity is normalized so `1.0` means neutral indoor lighting: +/// platform backends map their native scale to this convention (ARKit reports +/// about 1000 lumens in a neutral room, ARCore reports a mean pixel intensity +/// of about 0.18 - both map to `1.0`). Values below `1.0` mean the scene is +/// darker than neutral, values above mean brighter. +public final class ARLightEstimate { + /// An invalid estimate with neutral values, returned before the platform + /// produces its first real estimate. + public static final ARLightEstimate INVALID = + new ARLightEstimate(false, 1.0f, 1.0f, 1.0f, 1.0f); + + private final boolean valid; + private final float ambientIntensity; + private final float colorR; + private final float colorG; + private final float colorB; + + /// Creates a light estimate. Intended for platform implementations and + /// tests; applications receive estimates from the session. + /// + /// #### Parameters + /// + /// - `valid`: whether the platform produced a real estimate + /// + /// - `ambientIntensity`: normalized intensity, `1.0` is neutral + /// + /// - `colorR`, `colorG`, `colorB`: per-channel color correction scale + /// factors, `1.0` each is neutral white light + public ARLightEstimate(boolean valid, float ambientIntensity, + float colorR, float colorG, float colorB) { + this.valid = valid; + this.ambientIntensity = ambientIntensity; + this.colorR = colorR; + this.colorG = colorG; + this.colorB = colorB; + } + + /// True when the platform produced a real estimate; false for the + /// placeholder returned before tracking starts. + public boolean isValid() { + return valid; + } + + /// The normalized ambient light intensity. `1.0` is neutral indoor + /// lighting, lower is darker, higher is brighter. + public float getAmbientIntensity() { + return ambientIntensity; + } + + /// The per-channel color correction as a newly allocated `{r, g, b}` array + /// of scale factors. `{1, 1, 1}` is neutral white light; multiply your + /// content's color by these factors to match the scene's color cast. + public float[] getColorCorrection() { + return new float[]{colorR, colorG, colorB}; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARModel.java b/CodenameOne/src/com/codename1/ar/ARModel.java new file mode 100644 index 00000000000..f1799ad5d25 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARModel.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.gpu.GltfLoader; +import com.codename1.gpu.Mesh; +import com.codename1.ui.Image; + +import java.io.IOException; +import java.io.InputStream; + +/// The renderable content of an `ARNode`: geometry plus an optional base-color +/// texture or solid color. Models are immutable descriptors - the platform +/// backend uploads them into its native renderer when the node is attached to +/// an anchor. +/// +/// Create a model either from a glTF 2.0 asset (`.glb` or `.gltf` bytes, +/// parsed with `com.codename1.gpu.GltfLoader`) or directly from a +/// `com.codename1.gpu.Mesh` such as one built by +/// `com.codename1.gpu.Primitives`. Geometry units are meters in the anchor's +/// local frame. +public final class ARModel { + private final byte[] gltfBytes; + private Mesh mesh; + private Image baseColorImage; + private final int color; + private boolean parsed; + + private ARModel(byte[] gltfBytes, Mesh mesh, Image baseColorImage, int color) { + this.gltfBytes = gltfBytes; + this.mesh = mesh; + this.baseColorImage = baseColorImage; + this.color = color; + this.parsed = gltfBytes == null; + } + + /// Creates a model from in-memory glTF 2.0 bytes (binary `.glb` or JSON + /// `.gltf`). The geometry and base-color texture are parsed lazily on + /// first access. + /// + /// #### Parameters + /// + /// - `glbOrGltf`: the raw model bytes + /// + /// #### Returns + /// + /// the model descriptor + public static ARModel fromGltf(byte[] glbOrGltf) { + if (glbOrGltf == null || glbOrGltf.length == 0) { + throw new IllegalArgumentException("model bytes are required"); + } + byte[] copy = new byte[glbOrGltf.length]; + System.arraycopy(glbOrGltf, 0, copy, 0, glbOrGltf.length); + return new ARModel(copy, null, null, 0xffffffff); + } + + /// Reads all bytes from the stream and creates a glTF model. The stream is + /// closed. + /// + /// #### Parameters + /// + /// - `in`: a stream over `.glb` or `.gltf` bytes + /// + /// #### Returns + /// + /// the model descriptor + public static ARModel fromGltf(InputStream in) throws IOException { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + try { + byte[] buf = new byte[8192]; + int r; + while ((r = in.read(buf)) >= 0) { + out.write(buf, 0, r); + } + } finally { + com.codename1.io.Util.cleanup(in); + } + return fromGltf(out.toByteArray()); + } + + /// Creates a model from a mesh rendered in a single solid color. + /// + /// #### Parameters + /// + /// - `mesh`: the geometry, in meters + /// + /// - `argbColor`: the solid base color as `0xAARRGGBB` + /// + /// #### Returns + /// + /// the model descriptor + public static ARModel fromMesh(Mesh mesh, int argbColor) { + if (mesh == null) { + throw new IllegalArgumentException("mesh is required"); + } + return new ARModel(null, mesh, null, argbColor); + } + + /// Creates a model from a mesh with a base-color texture. + /// + /// #### Parameters + /// + /// - `mesh`: the geometry, in meters, with texture coordinates + /// + /// - `texture`: the base-color image + /// + /// #### Returns + /// + /// the model descriptor + public static ARModel fromMesh(Mesh mesh, Image texture) { + if (mesh == null) { + throw new IllegalArgumentException("mesh is required"); + } + if (texture == null) { + throw new IllegalArgumentException("texture is required"); + } + return new ARModel(null, mesh, texture, 0xffffffff); + } + + /// The raw glTF bytes as a newly allocated array, or null for mesh-based + /// models. + public byte[] getGltfBytes() { + if (gltfBytes == null) { + return null; + } + byte[] copy = new byte[gltfBytes.length]; + System.arraycopy(gltfBytes, 0, copy, 0, gltfBytes.length); + return copy; + } + + /// The model geometry. For glTF models the geometry is parsed on first + /// access. + public synchronized Mesh getMesh() { + parseIfNeeded(); + return mesh; + } + + /// The decoded base-color image, or null when the model has none. For + /// glTF models the image is extracted on first access. + public synchronized Image getBaseColorImage() { + parseIfNeeded(); + return baseColorImage; + } + + /// The solid base color as `0xAARRGGBB`, used when the model carries no + /// texture. Defaults to opaque white. + public int getColor() { + return color; + } + + private void parseIfNeeded() { + if (!parsed) { + GltfLoader.GltfImageModel m = GltfLoader.loadImageModel(gltfBytes); + mesh = m.getMesh(); + baseColorImage = m.getBaseColorImage(); + parsed = true; + } + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARNode.java b/CodenameOne/src/com/codename1/ar/ARNode.java new file mode 100644 index 00000000000..de0f5612408 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARNode.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import java.util.ArrayList; + +/// A node in the small scene graph rendered at an `ARAnchor`. A node carries +/// an optional `ARModel` plus a local transform relative to its parent (or to +/// the anchor for the root node), and may hold child nodes. +/// +/// Attach a root node to an anchor with `ARAnchor#setNode(ARNode)`. After +/// attachment, transform and visibility changes are forwarded to the platform +/// renderer automatically. Mutate nodes on the EDT. +public class ARNode { + private final ARModel model; + private float x; + private float y; + private float z; + private float qx; + private float qy; + private float qz; + private float qw = 1; + private float scale = 1; + private boolean visible = true; + private final ArrayList children = new ArrayList(); + private ARNode parent; + ARSession session; + String anchorId; + + /// Creates an empty grouping node with no geometry of its own. + public ARNode() { + this.model = null; + } + + /// Creates a node rendering the supplied model. + /// + /// #### Parameters + /// + /// - `model`: the content to render at this node + public ARNode(ARModel model) { + this.model = model; + } + + /// The model rendered at this node, or null for a grouping node. + public ARModel getModel() { + return model; + } + + /// Sets the node position relative to its parent (or the anchor for a + /// root node), in meters. + public ARNode setLocalPosition(float x, float y, float z) { + this.x = x; + this.y = y; + this.z = z; + changed(); + return this; + } + + /// Sets the node rotation relative to its parent as a quaternion. + public ARNode setLocalRotation(float qx, float qy, float qz, float qw) { + this.qx = qx; + this.qy = qy; + this.qz = qz; + this.qw = qw; + changed(); + return this; + } + + /// Sets a uniform scale factor relative to the parent. Default `1`. + public ARNode setLocalScale(float scale) { + this.scale = scale; + changed(); + return this; + } + + /// Shows or hides this node and its children. Default visible. + public ARNode setVisible(boolean visible) { + this.visible = visible; + changed(); + return this; + } + + /// The local X position in meters. + public float getLocalX() { + return x; + } + + /// The local Y position in meters. + public float getLocalY() { + return y; + } + + /// The local Z position in meters. + public float getLocalZ() { + return z; + } + + /// The X component of the local rotation quaternion. + public float getLocalQx() { + return qx; + } + + /// The Y component of the local rotation quaternion. + public float getLocalQy() { + return qy; + } + + /// The Z component of the local rotation quaternion. + public float getLocalQz() { + return qz; + } + + /// The W component of the local rotation quaternion. + public float getLocalQw() { + return qw; + } + + /// The uniform local scale factor. + public float getLocalScale() { + return scale; + } + + /// True when this node (and therefore its children) is rendered. + public boolean isVisible() { + return visible; + } + + /// Adds a child node. A node may have at most one parent. + /// + /// #### Parameters + /// + /// - `child`: the node to add below this one + public void addChild(ARNode child) { + if (child == null) { + throw new IllegalArgumentException("child is required"); + } + if (child.parent != null) { + throw new IllegalArgumentException("node already has a parent"); + } + child.parent = this; + children.add(child); + changed(); + } + + /// Removes a child node. No-op when the node is not a child of this one. + /// + /// #### Parameters + /// + /// - `child`: the node to remove + public void removeChild(ARNode child) { + if (children.remove(child)) { + child.parent = null; + changed(); + } + } + + /// The number of child nodes. + public int getChildCount() { + return children.size(); + } + + /// The child node at the supplied index. + public ARNode getChildAt(int index) { + return children.get(index); + } + + /// The parent node, or null for a root node. + public ARNode getParent() { + return parent; + } + + private void changed() { + ARNode root = this; + while (root.parent != null) { + root = root.parent; + } + if (root.session != null) { + root.session.nodeChangedInternal(root.anchorId, root); + } + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARPlane.java b/CodenameOne/src/com/codename1/ar/ARPlane.java new file mode 100644 index 00000000000..5fc9b26a61b --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARPlane.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// A real-world surface detected by the AR session, such as a floor, table or +/// wall. Planes grow and merge as the session learns more about the +/// environment; each refinement is delivered as a new immutable `ARPlane` +/// instance carrying the same `getId()`, through the session's +/// `ARPlaneListener`. +/// +/// The plane's local coordinate frame is centered at `getCenterPose()` with +/// the surface spanning the local X and Z axes and the normal along local Y. +public final class ARPlane { + /// The orientation of a detected plane. + public enum Type { + /// A horizontal, upward-facing surface such as a floor or table top. + HORIZONTAL_UP, + + /// A horizontal, downward-facing surface such as a ceiling. + HORIZONTAL_DOWN, + + /// A vertical surface such as a wall. + VERTICAL + } + + private final String id; + private final Type type; + private final ARPose centerPose; + private final float extentX; + private final float extentZ; + private final float[] polygon; + private final ARTrackingState trackingState; + + /// Creates a plane snapshot. Intended for platform implementations and + /// tests; applications receive planes from the session. + /// + /// #### Parameters + /// + /// - `id`: the stable identifier shared by all snapshots of this plane + /// + /// - `type`: the surface orientation + /// + /// - `centerPose`: the pose of the plane center in world space; local X/Z + /// span the surface, local Y is the normal + /// + /// - `extentX`: the surface extent along local X in meters + /// + /// - `extentZ`: the surface extent along local Z in meters + /// + /// - `polygon`: an optional boundary polygon as `x, z` pairs in the + /// plane's local frame, or null when the platform supplies none + /// + /// - `trackingState`: the tracking quality of this plane + public ARPlane(String id, Type type, ARPose centerPose, float extentX, float extentZ, + float[] polygon, ARTrackingState trackingState) { + if (id == null || id.length() == 0) { + throw new IllegalArgumentException("id is required"); + } + if (type == null) { + throw new IllegalArgumentException("type is required"); + } + if (centerPose == null) { + throw new IllegalArgumentException("centerPose is required"); + } + this.id = id; + this.type = type; + this.centerPose = centerPose; + this.extentX = extentX; + this.extentZ = extentZ; + if (polygon == null) { + this.polygon = null; + } else { + float[] copy = new float[polygon.length]; + System.arraycopy(polygon, 0, copy, 0, polygon.length); + this.polygon = copy; + } + this.trackingState = trackingState == null ? ARTrackingState.TRACKING : trackingState; + } + + /// The stable identifier shared by all snapshots of this physical plane. + public String getId() { + return id; + } + + /// The surface orientation. + public Type getType() { + return type; + } + + /// The pose of the plane center in world space. Local X/Z span the + /// surface, local Y is the surface normal. + public ARPose getCenterPose() { + return centerPose; + } + + /// The surface extent along the plane's local X axis in meters. + public float getExtentX() { + return extentX; + } + + /// The surface extent along the plane's local Z axis in meters. + public float getExtentZ() { + return extentZ; + } + + /// The boundary polygon as a newly allocated array of `x, z` pairs in the + /// plane's local frame, or null when the platform supplies none. + public float[] getPolygon() { + if (polygon == null) { + return null; + } + float[] copy = new float[polygon.length]; + System.arraycopy(polygon, 0, copy, 0, polygon.length); + return copy; + } + + /// The tracking quality of this plane. + public ARTrackingState getTrackingState() { + return trackingState; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARPlaneDetection.java b/CodenameOne/src/com/codename1/ar/ARPlaneDetection.java new file mode 100644 index 00000000000..0a801e6ca89 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARPlaneDetection.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Which real-world surface orientations an `ARSession` should detect and +/// report as `ARPlane`s. +public enum ARPlaneDetection { + /// Do not detect planes. + NONE, + + /// Detect horizontal surfaces such as floors and tables. + HORIZONTAL, + + /// Detect vertical surfaces such as walls. + VERTICAL, + + /// Detect both horizontal and vertical surfaces. + HORIZONTAL_AND_VERTICAL; + + /// True when this setting includes horizontal surfaces. + public boolean includesHorizontal() { + return this == HORIZONTAL || this == HORIZONTAL_AND_VERTICAL; + } + + /// True when this setting includes vertical surfaces. + public boolean includesVertical() { + return this == VERTICAL || this == HORIZONTAL_AND_VERTICAL; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARPlaneEvent.java b/CodenameOne/src/com/codename1/ar/ARPlaneEvent.java new file mode 100644 index 00000000000..e0a8bfb756f --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARPlaneEvent.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Describes a change to a detected `ARPlane`, delivered to +/// `ARPlaneListener`s on the EDT. +public final class ARPlaneEvent { + /// The kind of change that occurred. + public enum Kind { + /// The plane was newly detected. + ADDED, + + /// The plane's geometry or pose was refined; `#getPlane()` is the new + /// snapshot. + UPDATED, + + /// The plane was removed, for example merged into another plane. + REMOVED + } + + private final Kind kind; + private final ARPlane plane; + private final ARSession session; + + ARPlaneEvent(Kind kind, ARPlane plane, ARSession session) { + this.kind = kind; + this.plane = plane; + this.session = session; + } + + /// The kind of change that occurred. + public Kind getKind() { + return kind; + } + + /// The plane snapshot after the change. For `Kind#REMOVED` this is the + /// last known snapshot. + public ARPlane getPlane() { + return plane; + } + + /// The session that detected the change. + public ARSession getSession() { + return session; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARPlaneListener.java b/CodenameOne/src/com/codename1/ar/ARPlaneListener.java new file mode 100644 index 00000000000..599d724037a --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARPlaneListener.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Observes plane detection. Register through +/// `ARSession#addPlaneListener(ARPlaneListener)`; events are delivered on the +/// EDT. +public interface ARPlaneListener { + /// Invoked on the EDT when a plane is detected, refined or removed. + /// + /// #### Parameters + /// + /// - `event`: the change description + void planeChanged(ARPlaneEvent event); +} diff --git a/CodenameOne/src/com/codename1/ar/ARPose.java b/CodenameOne/src/com/codename1/ar/ARPose.java new file mode 100644 index 00000000000..66a239fc0ba --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARPose.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.gpu.Quaternion; + +/// An immutable rigid transform - a rotation followed by a translation - that +/// positions something in AR world space. Poses use the AR coordinate +/// convention shared by ARKit and ARCore: units are meters, the coordinate +/// system is right-handed with Y pointing up and -Z pointing forward from the +/// initial camera direction. +/// +/// The rotation is stored as a unit quaternion `(qx, qy, qz, qw)` compatible +/// with `com.codename1.gpu.Quaternion`; the translation is `(tx, ty, tz)`. +/// `toMatrix(float[])` produces the equivalent column-major 4x4 matrix, ready +/// for `com.codename1.gpu.Matrix4` math. +public final class ARPose { + /// The identity pose: no rotation, located at the world origin. + public static final ARPose IDENTITY = new ARPose(0, 0, 0, 0, 0, 0, 1); + + private final float tx; + private final float ty; + private final float tz; + private final float qx; + private final float qy; + private final float qz; + private final float qw; + + /// Creates a pose from a translation and a rotation quaternion. The + /// quaternion is normalized defensively; a zero quaternion becomes the + /// identity rotation. + /// + /// #### Parameters + /// + /// - `tx`, `ty`, `tz`: the translation in meters + /// + /// - `qx`, `qy`, `qz`, `qw`: the rotation quaternion + public ARPose(float tx, float ty, float tz, float qx, float qy, float qz, float qw) { + this.tx = tx; + this.ty = ty; + this.tz = tz; + float len = (float) Math.sqrt(qx * qx + qy * qy + qz * qz + qw * qw); + if (len == 0.0f) { + this.qx = 0; + this.qy = 0; + this.qz = 0; + this.qw = 1; + } else { + float inv = 1.0f / len; + this.qx = qx * inv; + this.qy = qy * inv; + this.qz = qz * inv; + this.qw = qw * inv; + } + } + + /// Extracts a pose from a column-major 4x4 transform matrix whose upper + /// left 3x3 is a pure rotation (no scale or shear). + /// + /// #### Parameters + /// + /// - `m16`: the column-major matrix, 16 floats + /// + /// #### Returns + /// + /// the equivalent pose + public static ARPose fromMatrix(float[] m16) { + float m00 = m16[0]; + float m10 = m16[1]; + float m20 = m16[2]; + float m01 = m16[4]; + float m11 = m16[5]; + float m21 = m16[6]; + float m02 = m16[8]; + float m12 = m16[9]; + float m22 = m16[10]; + float trace = m00 + m11 + m22; + float x; + float y; + float z; + float w; + if (trace > 0.0f) { + float s = (float) Math.sqrt(trace + 1.0f) * 2.0f; + w = 0.25f * s; + x = (m21 - m12) / s; + y = (m02 - m20) / s; + z = (m10 - m01) / s; + } else if (m00 > m11 && m00 > m22) { + float s = (float) Math.sqrt(1.0f + m00 - m11 - m22) * 2.0f; + w = (m21 - m12) / s; + x = 0.25f * s; + y = (m01 + m10) / s; + z = (m02 + m20) / s; + } else if (m11 > m22) { + float s = (float) Math.sqrt(1.0f + m11 - m00 - m22) * 2.0f; + w = (m02 - m20) / s; + x = (m01 + m10) / s; + y = 0.25f * s; + z = (m12 + m21) / s; + } else { + float s = (float) Math.sqrt(1.0f + m22 - m00 - m11) * 2.0f; + w = (m10 - m01) / s; + x = (m02 + m20) / s; + y = (m12 + m21) / s; + z = 0.25f * s; + } + return new ARPose(m16[12], m16[13], m16[14], x, y, z, w); + } + + /// The X component of the translation in meters. + public float getTx() { + return tx; + } + + /// The Y component of the translation in meters. + public float getTy() { + return ty; + } + + /// The Z component of the translation in meters. + public float getTz() { + return tz; + } + + /// The X component of the rotation quaternion. + public float getQx() { + return qx; + } + + /// The Y component of the rotation quaternion. + public float getQy() { + return qy; + } + + /// The Z component of the rotation quaternion. + public float getQz() { + return qz; + } + + /// The W component of the rotation quaternion. + public float getQw() { + return qw; + } + + /// Writes this pose as a column-major 4x4 transform matrix into `out16`, + /// compatible with `com.codename1.gpu.Matrix4`. + /// + /// #### Parameters + /// + /// - `out16`: the destination array, 16 floats + public void toMatrix(float[] out16) { + float[] q = {qx, qy, qz, qw}; + Quaternion.toMatrix(q, out16); + out16[12] = tx; + out16[13] = ty; + out16[14] = tz; + } + + /// Returns this pose as a newly allocated column-major 4x4 transform + /// matrix. + public float[] toMatrix() { + float[] m = new float[16]; + toMatrix(m); + return m; + } + + /// Composes this pose with a pose expressed in this pose's local frame, + /// returning `this * local`. Use it to convert an offset relative to an + /// anchor into world space. + /// + /// #### Parameters + /// + /// - `local`: the pose in this pose's local coordinate frame + /// + /// #### Returns + /// + /// the composed pose in the frame this pose is expressed in + public ARPose transform(ARPose local) { + float[] q = {qx, qy, qz, qw}; + float[] lq = {local.qx, local.qy, local.qz, local.qw}; + float[] rq = new float[4]; + Quaternion.multiply(q, lq, rq); + float[] t = {local.tx, local.ty, local.tz}; + Quaternion.rotateVector(q, t); + return new ARPose(tx + t[0], ty + t[1], tz + t[2], rq[0], rq[1], rq[2], rq[3]); + } + + /// Transforms the point stored in `xyzInOut` (3 floats) from this pose's + /// local frame to the frame this pose is expressed in, writing the result + /// back in place. + /// + /// #### Parameters + /// + /// - `xyzInOut`: the point to transform, modified in place + public void transformPoint(float[] xyzInOut) { + float[] q = {qx, qy, qz, qw}; + Quaternion.rotateVector(q, xyzInOut); + xyzInOut[0] += tx; + xyzInOut[1] += ty; + xyzInOut[2] += tz; + } + + @Override + public String toString() { + return "ARPose(t=[" + tx + ", " + ty + ", " + tz + "], q=[" + + qx + ", " + qy + ", " + qz + ", " + qw + "])"; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARReferenceImage.java b/CodenameOne/src/com/codename1/ar/ARReferenceImage.java new file mode 100644 index 00000000000..8caf81a364a --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARReferenceImage.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.ui.EncodedImage; +import com.codename1.ui.Image; + +/// A 2D image the AR session should recognize in the real world - a poster, a +/// game board, a product label. Register reference images through +/// `ARSessionOptions#referenceImages(ARReferenceImage[])`; when the camera +/// sees one, the session delivers an `ARImageAnchor` carrying this image's +/// name. +/// +/// The physical width tells the platform how large the printed image is in the +/// real world, which it needs to estimate the image's distance and pose +/// accurately. +public final class ARReferenceImage { + private final String name; + private final byte[] encodedImage; + private final float physicalWidthMeters; + + /// Creates a reference image from encoded (PNG or JPEG) bytes. + /// + /// #### Parameters + /// + /// - `name`: a unique name identifying this image in `ARImageAnchor`s + /// + /// - `encodedImage`: the PNG or JPEG bytes of the image to detect + /// + /// - `physicalWidthMeters`: the width of the printed image in meters + public ARReferenceImage(String name, byte[] encodedImage, float physicalWidthMeters) { + if (name == null || name.length() == 0) { + throw new IllegalArgumentException("name is required"); + } + if (encodedImage == null || encodedImage.length == 0) { + throw new IllegalArgumentException("encodedImage is required"); + } + if (physicalWidthMeters <= 0.0f) { + throw new IllegalArgumentException("physicalWidthMeters must be positive"); + } + this.name = name; + byte[] copy = new byte[encodedImage.length]; + System.arraycopy(encodedImage, 0, copy, 0, encodedImage.length); + this.encodedImage = copy; + this.physicalWidthMeters = physicalWidthMeters; + } + + /// Creates a reference image from an `Image`, encoding it as PNG. + /// + /// #### Parameters + /// + /// - `name`: a unique name identifying this image in `ARImageAnchor`s + /// + /// - `image`: the image to detect + /// + /// - `physicalWidthMeters`: the width of the printed image in meters + public ARReferenceImage(String name, Image image, float physicalWidthMeters) { + this(name, encode(image), physicalWidthMeters); + } + + private static byte[] encode(Image image) { + if (image == null) { + throw new IllegalArgumentException("image is required"); + } + if (image instanceof EncodedImage) { + return ((EncodedImage) image).getImageData(); + } + return EncodedImage.createFromImage(image, false).getImageData(); + } + + /// The unique name identifying this image. + public String getName() { + return name; + } + + /// The encoded (PNG or JPEG) bytes of the image as a newly allocated + /// array. + public byte[] getEncodedImage() { + byte[] copy = new byte[encodedImage.length]; + System.arraycopy(encodedImage, 0, copy, 0, encodedImage.length); + return copy; + } + + /// The width of the printed image in meters. + public float getPhysicalWidthMeters() { + return physicalWidthMeters; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARSession.java b/CodenameOne/src/com/codename1/ar/ARSession.java new file mode 100644 index 00000000000..6b237517856 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARSession.java @@ -0,0 +1,612 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.impl.ARImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.Map; + +/// Active augmented reality session. Obtained from +/// `AR#open(ARSessionOptions)`. +/// +/// Only one `ARSession` may be open at a time; opening a second throws +/// `IllegalStateException`. Closing the session releases the camera and +/// tracking hardware and invalidates the `ARView` returned by +/// `#createView()`. +/// +/// All events are delivered on the EDT and every getter reflects the state as +/// of the most recently delivered events. High-frequency refinements (anchor +/// and plane updates, camera pose, light estimate) are coalesced so the EDT +/// sees the latest value rather than a backlog. +public final class ARSession implements AutoCloseable { + private final ARImpl impl; + private final ARSessionOptions options; + private final Bridge bridge = new Bridge(); + private ARView view; + private boolean closed; + + private final LinkedHashMap planes = new LinkedHashMap(); + private final LinkedHashMap anchors = new LinkedHashMap(); + private ARTrackingState trackingState = ARTrackingState.NOT_TRACKING; + private ARTrackingFailureReason failureReason = ARTrackingFailureReason.INITIALIZING; + private ARPose cameraPose = ARPose.IDENTITY; + private ARLightEstimate lightEstimate = ARLightEstimate.INVALID; + + private final ArrayList trackingListeners = new ArrayList(); + private final ArrayList planeListeners = new ArrayList(); + private final ArrayList anchorListeners = new ArrayList(); + + ARSession(ARImpl impl, ARSessionOptions options) { + this.impl = impl; + this.options = options; + impl.setEventSink(bridge); + } + + /// Creates the AR view that renders the camera image composited with the + /// anchored content. Each session owns one view; subsequent calls return + /// the same instance. + public ARView createView() { + if (view == null) { + view = new ARView(this, impl.createViewPeer()); + } + return view; + } + + /// The options the session was opened with. Read-only snapshot; mutating + /// it after `AR#open(ARSessionOptions)` has no effect. + public ARSessionOptions getOptions() { + return options; + } + + /// The session's overall tracking quality as of the latest update. + public ARTrackingState getTrackingState() { + return trackingState; + } + + /// Why tracking is degraded, or `ARTrackingFailureReason#NONE` when it is + /// not. + public ARTrackingFailureReason getTrackingFailureReason() { + return failureReason; + } + + /// The latest device pose in world space. Poll-style: the value refreshes + /// every frame without firing events. + public ARPose getCameraPose() { + return cameraPose; + } + + /// The latest estimate of real-world lighting. Poll-style: the value + /// refreshes every frame without firing events. Before the first platform + /// estimate this returns `ARLightEstimate#INVALID`. + public ARLightEstimate getLightEstimate() { + return lightEstimate; + } + + /// Performs an asynchronous hit test from a normalized view coordinate + /// (`0.0` top left to `1.0` bottom right) into the world. The returned + /// `AsyncResource` resolves on the EDT with the intersections ordered + /// nearest first; an empty array means nothing was hit. + /// + /// #### Parameters + /// + /// - `xNorm`: the horizontal view coordinate, `0.0` to `1.0` + /// + /// - `yNorm`: the vertical view coordinate, `0.0` to `1.0` + /// + /// #### Returns + /// + /// resolves with the hits; call `ARHitResult#createAnchor()` on one to + /// place content there + public AsyncResource hitTest(float xNorm, float yNorm) { + checkClosed(); + final AsyncResource out = new AsyncResource(); + AsyncResource in = new AsyncResource(); + in.ready(new SuccessCallback() { + @Override public void onSucess(ARHitResult[] hits) { + if (hits == null) { + hits = new ARHitResult[0]; + } + for (int i = 0; i < hits.length; i++) { + hits[i].session = ARSession.this; + } + out.complete(hits); + } + }); + in.except(new ErrorForwarder(out)); + impl.hitTest(xNorm, yNorm, in); + return out; + } + + /// Forwards an implementation hit-test failure to the application-facing + /// resource. A named static class since it only needs the target resource, + /// not the enclosing session. + private static final class ErrorForwarder implements SuccessCallback { + private final AsyncResource out; + + ErrorForwarder(AsyncResource out) { + this.out = out; + } + + @Override + public void onSucess(Throwable t) { + out.error(t); + } + } + + /// Creates an anchor at an arbitrary world pose and registers it with the + /// session. Prefer `ARHitResult#createAnchor()` when placing content on + /// detected geometry. + /// + /// #### Parameters + /// + /// - `pose`: the world pose to anchor + /// + /// #### Returns + /// + /// the new anchor + public ARAnchor createAnchor(ARPose pose) { + checkClosed(); + if (pose == null) { + throw new IllegalArgumentException("pose is required"); + } + return registerAnchor(impl.createAnchor(pose), pose); + } + + ARAnchor createAnchorFromHitInternal(ARHitResult hit) { + checkClosed(); + return registerAnchor(impl.createAnchorFromHit(hit.getNativeHandle(), hit.getPose()), + hit.getPose()); + } + + private ARAnchor registerAnchor(String id, ARPose pose) { + ARAnchor anchor = new ARAnchor(id, pose); + anchor.session = this; + anchors.put(id, anchor); + fireAnchorEvent(new ARAnchorEvent(ARAnchorEvent.Kind.ADDED, anchor, this)); + return anchor; + } + + /// The anchors currently registered with the session, in registration + /// order. + public ARAnchor[] getAnchors() { + return anchors.values().toArray(new ARAnchor[anchors.size()]); + } + + /// The planes the session currently tracks, in detection order. + public ARPlane[] getPlanes() { + return planes.values().toArray(new ARPlane[planes.size()]); + } + + /// Registers a tracking state listener. Events fire on the EDT. + public void addTrackingListener(ARTrackingListener l) { + if (l != null && !trackingListeners.contains(l)) { + trackingListeners.add(l); + } + } + + /// Removes a tracking state listener. + public void removeTrackingListener(ARTrackingListener l) { + trackingListeners.remove(l); + } + + /// Registers a plane listener. Events fire on the EDT. + public void addPlaneListener(ARPlaneListener l) { + if (l != null && !planeListeners.contains(l)) { + planeListeners.add(l); + } + } + + /// Removes a plane listener. + public void removePlaneListener(ARPlaneListener l) { + planeListeners.remove(l); + } + + /// Registers an anchor listener. Events fire on the EDT. + public void addAnchorListener(ARAnchorListener l) { + if (l != null && !anchorListeners.contains(l)) { + anchorListeners.add(l); + } + } + + /// Removes an anchor listener. + public void removeAnchorListener(ARAnchorListener l) { + anchorListeners.remove(l); + } + + /// Suspends tracking and the camera while keeping this session object + /// alive. Pair with `#resume()`. + public void pause() { + if (!closed) { + impl.pause(); + } + } + + /// Re-acquires the camera and resumes tracking after `#pause()`. + public void resume() { + if (!closed) { + impl.resume(); + } + } + + /// Releases the session. Idempotent. + @Override + public void close() { + if (closed) { + return; + } + closed = true; + bridge.stop(); + try { + impl.close(); + } finally { + AR.clearActive(this); + } + } + + /// True once `#close()` has been called on this session. + public boolean isClosed() { + return closed; + } + + void anchorDetachedInternal(ARAnchor anchor) { + if (!closed) { + impl.removeAnchor(anchor.getId()); + } + if (anchors.remove(anchor.getId()) != null) { + fireAnchorEvent(new ARAnchorEvent(ARAnchorEvent.Kind.REMOVED, anchor, this)); + } + } + + void anchorNodeChangedInternal(ARAnchor anchor, ARNode node) { + if (!closed) { + impl.setAnchorNode(anchor.getId(), node); + } + } + + void nodeChangedInternal(String anchorId, ARNode root) { + if (!closed && anchorId != null) { + impl.nodeChanged(anchorId, root); + } + } + + ARImpl getImpl() { + return impl; + } + + private void checkClosed() { + if (closed) { + throw new IllegalStateException("the AR session is closed"); + } + } + + private void fireTrackingEvent() { + ARTrackingListener[] ls = trackingListeners.toArray( + new ARTrackingListener[trackingListeners.size()]); + for (ARTrackingListener l : ls) { + l.trackingStateChanged(this, trackingState, failureReason); + } + } + + private void firePlaneEvent(ARPlaneEvent ev) { + ARPlaneListener[] ls = planeListeners.toArray( + new ARPlaneListener[planeListeners.size()]); + for (ARPlaneListener l : ls) { + l.planeChanged(ev); + } + } + + private void fireAnchorEvent(ARAnchorEvent ev) { + ARAnchorListener[] ls = anchorListeners.toArray( + new ARAnchorListener[anchorListeners.size()]); + for (ARAnchorListener l : ls) { + l.anchorChanged(ev); + } + } + + private static final int OP_TRACKING = 0; + private static final int OP_PLANE_ADDED = 1; + private static final int OP_PLANE_REMOVED = 2; + private static final int OP_ANCHOR_ADDED = 3; + private static final int OP_ANCHOR_REMOVED = 4; + + /// Marshals implementation events - which may arrive on any thread - to + /// the EDT. Discrete events (add/remove/tracking) keep their order; + /// per-frame refinements coalesce to the latest value per id so a slow + /// EDT never accumulates a backlog. + private final class Bridge implements ARImpl.EventSink { + private final Object lock = new Object(); + private final ArrayList ops = new ArrayList(); + private final LinkedHashMap planeUpdates = + new LinkedHashMap(); + private final LinkedHashMap anchorUpdates = + new LinkedHashMap(); + private ARPose pendingCameraPose; + private ARLightEstimate pendingLight; + private boolean drainScheduled; + // Set under the lock when the session closes; every producer checks + // it so late implementation events are dropped without touching the + // (EDT-confined) session state. + private boolean stopped; + + void stop() { + synchronized (lock) { + stopped = true; + ops.clear(); + planeUpdates.clear(); + anchorUpdates.clear(); + pendingCameraPose = null; + pendingLight = null; + } + } + + @Override + public void onTrackingStateChanged(ARTrackingState state, ARTrackingFailureReason reason) { + enqueueOp(new Object[]{Integer.valueOf(OP_TRACKING), state, reason}); + } + + @Override + public void onPlaneAdded(ARPlane plane) { + if (plane != null) { + enqueueOp(new Object[]{Integer.valueOf(OP_PLANE_ADDED), plane}); + } + } + + @Override + public void onPlaneUpdated(ARPlane plane) { + if (plane == null) { + return; + } + synchronized (lock) { + if (stopped) { + return; + } + planeUpdates.put(plane.getId(), plane); + scheduleDrain(); + } + } + + @Override + public void onPlaneRemoved(String planeId) { + if (planeId != null) { + enqueueOp(new Object[]{Integer.valueOf(OP_PLANE_REMOVED), planeId}); + } + } + + @Override + public void onAnchorAdded(ARAnchor anchor) { + if (anchor != null) { + enqueueOp(new Object[]{Integer.valueOf(OP_ANCHOR_ADDED), anchor}); + } + } + + @Override + public void onAnchorUpdated(String anchorId, ARPose pose, ARTrackingState state) { + onFaceAnchorUpdated(anchorId, pose, state, null, null, null); + } + + @Override + public void onFaceAnchorUpdated(String anchorId, ARPose pose, ARTrackingState state, + ARPose[] regionPoses, float[] meshVertices, + int[] meshTriangles) { + if (anchorId == null) { + return; + } + synchronized (lock) { + if (stopped) { + return; + } + Object[] prev = anchorUpdates.get(anchorId); + if (prev != null) { + // Merge with the pending update so face payloads survive a + // later pose-only refinement in the same batch. + if (pose == null) { + pose = (ARPose) prev[0]; + } + if (state == null) { + state = (ARTrackingState) prev[1]; + } + if (regionPoses == null) { + regionPoses = (ARPose[]) prev[2]; + } + if (meshVertices == null) { + meshVertices = (float[]) prev[3]; + } + if (meshTriangles == null) { + meshTriangles = (int[]) prev[4]; + } + } + anchorUpdates.put(anchorId, + new Object[]{pose, state, regionPoses, meshVertices, meshTriangles}); + scheduleDrain(); + } + } + + @Override + public void onAnchorRemoved(String anchorId) { + if (anchorId != null) { + enqueueOp(new Object[]{Integer.valueOf(OP_ANCHOR_REMOVED), anchorId}); + } + } + + @Override + public void onLightEstimate(ARLightEstimate estimate) { + if (estimate == null) { + return; + } + synchronized (lock) { + if (stopped) { + return; + } + pendingLight = estimate; + scheduleDrain(); + } + } + + @Override + public void onCameraPose(ARPose pose) { + if (pose == null) { + return; + } + synchronized (lock) { + if (stopped) { + return; + } + pendingCameraPose = pose; + scheduleDrain(); + } + } + + private void enqueueOp(Object[] op) { + synchronized (lock) { + if (stopped) { + return; + } + ops.add(op); + scheduleDrain(); + } + } + + private void scheduleDrain() { + if (drainScheduled) { + return; + } + drainScheduled = true; + Display.getInstance().callSerially(new Runnable() { + @Override public void run() { + drain(); + } + }); + } + + private void drain() { + Object[][] opsSnapshot; + ARPlane[] planeSnapshot; + Map.Entry[] anchorSnapshot; + ARPose newCameraPose; + ARLightEstimate newLight; + synchronized (lock) { + drainScheduled = false; + if (stopped) { + return; + } + opsSnapshot = ops.toArray(new Object[ops.size()][]); + ops.clear(); + planeSnapshot = planeUpdates.values().toArray(new ARPlane[planeUpdates.size()]); + planeUpdates.clear(); + anchorSnapshot = anchorUpdates.entrySet().toArray( + new Map.Entry[anchorUpdates.size()]); + anchorUpdates.clear(); + newCameraPose = pendingCameraPose; + pendingCameraPose = null; + newLight = pendingLight; + pendingLight = null; + } + if (newCameraPose != null) { + cameraPose = newCameraPose; + } + if (newLight != null) { + lightEstimate = newLight; + } + for (Object[] op : opsSnapshot) { + applyOp(op); + } + for (ARPlane p : planeSnapshot) { + if (planes.containsKey(p.getId())) { + planes.put(p.getId(), p); + firePlaneEvent(new ARPlaneEvent(ARPlaneEvent.Kind.UPDATED, p, ARSession.this)); + } + } + for (Map.Entry entry : anchorSnapshot) { + String id = (String) entry.getKey(); + Object[] u = (Object[]) entry.getValue(); + ARAnchor anchor = anchors.get(id); + if (anchor == null) { + continue; + } + anchor.update((ARPose) u[0], (ARTrackingState) u[1]); + if (anchor instanceof ARFaceAnchor) { + ((ARFaceAnchor) anchor).updateFace( + (ARPose[]) u[2], (float[]) u[3], (int[]) u[4]); + } + fireAnchorEvent(new ARAnchorEvent(ARAnchorEvent.Kind.UPDATED, anchor, + ARSession.this)); + } + } + + private void applyOp(Object[] op) { + switch (((Integer) op[0]).intValue()) { + case OP_TRACKING: { + ARTrackingState state = (ARTrackingState) op[1]; + ARTrackingFailureReason reason = (ARTrackingFailureReason) op[2]; + trackingState = state == null ? ARTrackingState.NOT_TRACKING : state; + failureReason = reason == null ? ARTrackingFailureReason.NONE : reason; + fireTrackingEvent(); + break; + } + case OP_PLANE_ADDED: { + ARPlane plane = (ARPlane) op[1]; + boolean known = planes.containsKey(plane.getId()); + planes.put(plane.getId(), plane); + firePlaneEvent(new ARPlaneEvent( + known ? ARPlaneEvent.Kind.UPDATED : ARPlaneEvent.Kind.ADDED, + plane, ARSession.this)); + break; + } + case OP_PLANE_REMOVED: { + ARPlane plane = planes.remove((String) op[1]); + if (plane != null) { + firePlaneEvent(new ARPlaneEvent(ARPlaneEvent.Kind.REMOVED, plane, + ARSession.this)); + } + break; + } + case OP_ANCHOR_ADDED: { + ARAnchor anchor = (ARAnchor) op[1]; + if (!anchors.containsKey(anchor.getId())) { + anchor.session = ARSession.this; + anchors.put(anchor.getId(), anchor); + fireAnchorEvent(new ARAnchorEvent(ARAnchorEvent.Kind.ADDED, anchor, + ARSession.this)); + } + break; + } + case OP_ANCHOR_REMOVED: { + ARAnchor anchor = anchors.remove((String) op[1]); + if (anchor != null) { + anchor.markDetached(); + fireAnchorEvent(new ARAnchorEvent(ARAnchorEvent.Kind.REMOVED, anchor, + ARSession.this)); + } + break; + } + default: + break; + } + } + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARSessionOptions.java b/CodenameOne/src/com/codename1/ar/ARSessionOptions.java new file mode 100644 index 00000000000..3d5ea71ddcc --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARSessionOptions.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Configuration for an `ARSession`. Fluent builder. +/// +/// The defaults open a world tracking session that detects horizontal planes +/// and estimates lighting, which suits the common place-content-on-a-surface +/// use case. +public final class ARSessionOptions { + private ARTrackingMode trackingMode = ARTrackingMode.WORLD; + private ARPlaneDetection planeDetection = ARPlaneDetection.HORIZONTAL; + private boolean lightEstimation = true; + private ARReferenceImage[] referenceImages = new ARReferenceImage[0]; + + /// The tracking configuration to run. Default `ARTrackingMode#WORLD`. + public ARSessionOptions trackingMode(ARTrackingMode mode) { + this.trackingMode = mode == null ? ARTrackingMode.WORLD : mode; + return this; + } + + /// Which surface orientations to detect. Default + /// `ARPlaneDetection#HORIZONTAL`. Ignored in face tracking sessions. + public ARSessionOptions planeDetection(ARPlaneDetection detection) { + this.planeDetection = detection == null ? ARPlaneDetection.HORIZONTAL : detection; + return this; + } + + /// Whether to estimate real-world lighting. Default true. + public ARSessionOptions lightEstimation(boolean on) { + this.lightEstimation = on; + return this; + } + + /// Reference images the session should detect. When any are registered, + /// detected images are delivered as `ARImageAnchor`s through the anchor + /// listener. Default none. + public ARSessionOptions referenceImages(ARReferenceImage[] images) { + if (images == null) { + this.referenceImages = new ARReferenceImage[0]; + } else { + ARReferenceImage[] copy = new ARReferenceImage[images.length]; + System.arraycopy(images, 0, copy, 0, images.length); + this.referenceImages = copy; + } + return this; + } + + /// The tracking configuration to run. + public ARTrackingMode getTrackingMode() { + return trackingMode; + } + + /// Which surface orientations the session detects. + public ARPlaneDetection getPlaneDetection() { + return planeDetection; + } + + /// True when the session estimates real-world lighting. + public boolean isLightEstimation() { + return lightEstimation; + } + + /// The registered reference images as a newly allocated array. + public ARReferenceImage[] getReferenceImages() { + ARReferenceImage[] copy = new ARReferenceImage[referenceImages.length]; + System.arraycopy(referenceImages, 0, copy, 0, referenceImages.length); + return copy; + } +} diff --git a/CodenameOne/src/com/codename1/ar/ARTrackingFailureReason.java b/CodenameOne/src/com/codename1/ar/ARTrackingFailureReason.java new file mode 100644 index 00000000000..dce0e10c253 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARTrackingFailureReason.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Why an `ARSession` reports `ARTrackingState#LIMITED` or +/// `ARTrackingState#NOT_TRACKING`. Useful for prompting the user, for example +/// to move the device more slowly or to point it at a textured surface. +public enum ARTrackingFailureReason { + /// Tracking is not degraded. + NONE, + + /// The session is still gathering its first observations; tracking quality + /// improves as the user moves the device. + INITIALIZING, + + /// The device is moving too fast for reliable tracking. + EXCESSIVE_MOTION, + + /// The scene is too dark for the camera to track features. + INSUFFICIENT_LIGHT, + + /// The camera sees too few visual features, for example a blank wall. + INSUFFICIENT_FEATURES +} diff --git a/CodenameOne/src/com/codename1/ar/ARTrackingListener.java b/CodenameOne/src/com/codename1/ar/ARTrackingListener.java new file mode 100644 index 00000000000..b02c30090b7 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARTrackingListener.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// Observes changes to a session's overall tracking quality. Register through +/// `ARSession#addTrackingListener(ARTrackingListener)`; events are delivered +/// on the EDT. +public interface ARTrackingListener { + /// Invoked on the EDT when the session's tracking state changes. + /// + /// #### Parameters + /// + /// - `session`: the session whose state changed + /// + /// - `state`: the new tracking state + /// + /// - `reason`: why tracking is degraded, or + /// `ARTrackingFailureReason#NONE` when it is not + void trackingStateChanged(ARSession session, ARTrackingState state, + ARTrackingFailureReason reason); +} diff --git a/CodenameOne/src/com/codename1/ar/ARTrackingMode.java b/CodenameOne/src/com/codename1/ar/ARTrackingMode.java new file mode 100644 index 00000000000..1072a2df3ef --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARTrackingMode.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// The tracking configuration of an `ARSession`. World and face tracking are +/// distinct session configurations on both ARKit and ARCore, so a session runs +/// in exactly one mode; close the session and open a new one to switch. +public enum ARTrackingMode { + /// Track the device pose in the world using the rear camera. Enables plane + /// detection, hit testing, world anchors and image tracking. + WORLD, + + /// Track faces using the front camera. Detected faces are delivered as + /// `ARFaceAnchor`s; plane detection and hit testing are unavailable. + FACE +} diff --git a/CodenameOne/src/com/codename1/ar/ARTrackingState.java b/CodenameOne/src/com/codename1/ar/ARTrackingState.java new file mode 100644 index 00000000000..82dd9a73711 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARTrackingState.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +/// The quality of pose tracking for an `ARSession` or an individual trackable +/// such as an `ARAnchor` or `ARPlane`. +public enum ARTrackingState { + /// Tracking is unavailable; poses are stale or meaningless. + NOT_TRACKING, + + /// Tracking is running with degraded quality; see + /// `ARSession#getTrackingFailureReason()` for the cause. + LIMITED, + + /// Tracking is running normally. + TRACKING +} diff --git a/CodenameOne/src/com/codename1/ar/ARView.java b/CodenameOne/src/com/codename1/ar/ARView.java new file mode 100644 index 00000000000..f7fcb54cd5d --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/ARView.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.ui.Container; +import com.codename1.ui.PeerComponent; +import com.codename1.ui.layouts.BorderLayout; + +/// Component that renders the AR camera image composited with the session's +/// anchored 3D content. Created via `ARSession#createView()` and added to a +/// form like any other component. +/// +/// To place content where the user taps, convert the pointer coordinate to +/// the normalized view coordinate expected by +/// `ARSession#hitTest(float, float)`: +/// +/// ```java +/// view.addPointerReleasedListener(e -> { +/// float xn = (e.getX() - view.getAbsoluteX()) / (float) view.getWidth(); +/// float yn = (e.getY() - view.getAbsoluteY()) / (float) view.getHeight(); +/// session.hitTest(xn, yn).ready(hits -> { ... }); +/// }); +/// ``` +public final class ARView extends Container { + private final ARSession session; + private final PeerComponent peer; + + ARView(ARSession session, PeerComponent peer) { + super(new BorderLayout()); + this.session = session; + this.peer = peer; + if (peer != null) { + add(BorderLayout.CENTER, peer); + } + } + + /// The `ARSession` backing this view. + public ARSession getSession() { + return session; + } + + /// Exposed for ports that need to reach the native view directly. + public PeerComponent getViewPeer() { + return peer; + } +} diff --git a/CodenameOne/src/com/codename1/ar/package-info.java b/CodenameOne/src/com/codename1/ar/package-info.java new file mode 100644 index 00000000000..ab97d1828c4 --- /dev/null +++ b/CodenameOne/src/com/codename1/ar/package-info.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/// Cross-platform augmented reality: world tracking, plane detection, hit +/// testing, anchors, 3D content placement, light estimation, image tracking +/// and face tracking, backed by ARKit on iOS and ARCore on Android. +/// +/// Start at `AR`: check `AR#isSupported()` and per-feature +/// `AR#getCapabilities()`, then `AR#open(ARSessionOptions)` a session and add +/// its `ARSession#createView()` to a form. Place content by hit testing +/// screen taps (`ARSession#hitTest(float, float)`), creating an `ARAnchor` at +/// a hit and attaching an `ARNode` holding an `ARModel` (glTF or +/// `com.codename1.gpu.Mesh` geometry). +/// +/// **Coordinates**: world space is right-handed, in meters, with Y up and -Z +/// forward from the initial camera direction - the convention shared by ARKit +/// and ARCore. Poses are `ARPose` (translation + quaternion), convertible to +/// `com.codename1.gpu.Matrix4` compatible matrices. +/// +/// **Threading**: all listeners fire on the EDT and all getters reflect the +/// latest delivered state. High-frequency refinements are coalesced. +/// +/// **Permissions and dependencies**: referencing this package makes the build +/// pipeline inject the camera permission and plist usage description plus the +/// platform AR dependency into the application automatically; apps that do +/// not use AR pay no cost. Devices without AR hardware report +/// `AR#isSupported()` false, as do the simulator-less desktop targets - the +/// Codename One simulator itself ships a simulated AR environment for +/// development. +package com.codename1.ar; diff --git a/CodenameOne/src/com/codename1/gpu/GltfLoader.java b/CodenameOne/src/com/codename1/gpu/GltfLoader.java index 790c450d76b..bc5b1f636d4 100644 --- a/CodenameOne/src/com/codename1/gpu/GltfLoader.java +++ b/CodenameOne/src/com/codename1/gpu/GltfLoader.java @@ -95,8 +95,43 @@ public static Mesh load(GraphicsDevice device, InputStream in) throws IOExceptio /// /// the loaded mesh public static Mesh load(GraphicsDevice device, byte[] data) { + return load(data); + } + + /// Loads a model from in-memory `.glb` or `.gltf` bytes without a + /// `GraphicsDevice`, allocating the buffers directly. Behaves exactly like + /// `load(GraphicsDevice, byte[])`; useful for parsing geometry off the + /// render thread or handing meshes to non GPU consumers such as the AR + /// content pipeline. + /// + /// #### Parameters + /// + /// - `data`: the raw model bytes (binary `.glb` or JSON `.gltf`) + /// + /// #### Returns + /// + /// the loaded mesh + public static Mesh load(byte[] data) { Object[] parsed = parse(data); - return build(device, (Map) parsed[0], (byte[]) parsed[1]); + return build((Map) parsed[0], (byte[]) parsed[1]); + } + + /// Reads all bytes from the stream and loads the model without a + /// `GraphicsDevice`. The stream is closed. + /// + /// #### Parameters + /// + /// - `in`: a stream over `.glb` or `.gltf` bytes + /// + /// #### Returns + /// + /// the loaded mesh + public static Mesh load(InputStream in) throws IOException { + try { + return load(readFully(in)); + } finally { + Util.cleanup(in); + } } /// Loads a model together with its base-color texture from in-memory `.glb` @@ -117,7 +152,7 @@ public static GltfModel loadModel(GraphicsDevice device, byte[] data) { Object[] parsed = parse(data); Map root = (Map) parsed[0]; byte[] binChunk = (byte[]) parsed[1]; - Mesh mesh = build(device, root, binChunk); + Mesh mesh = build(root, binChunk); Texture baseColor = loadBaseColorTexture(device, root, binChunk); return new GltfModel(mesh, baseColor); } @@ -132,6 +167,38 @@ public static GltfModel loadModel(GraphicsDevice device, InputStream in) throws } } + /// Loads a model together with its base-color image from in-memory `.glb` + /// or `.gltf` bytes without a `GraphicsDevice`. The image is decoded but not + /// uploaded to the GPU, so this works off the render thread and on non GPU + /// consumers such as the AR content pipeline. + /// + /// #### Parameters + /// + /// - `data`: the raw model bytes + /// + /// #### Returns + /// + /// the loaded mesh plus its decoded base-color image (null image if the + /// model has none) + public static GltfImageModel loadImageModel(byte[] data) { + Object[] parsed = parse(data); + Map root = (Map) parsed[0]; + byte[] binChunk = (byte[]) parsed[1]; + Mesh mesh = build(root, binChunk); + Image baseColor = readBaseColorImage(root, binChunk); + return new GltfImageModel(mesh, baseColor); + } + + /// Reads all bytes from the stream and loads the model with its base-color + /// image without a `GraphicsDevice`. The stream is closed. + public static GltfImageModel loadImageModel(InputStream in) throws IOException { + try { + return loadImageModel(readFully(in)); + } finally { + Util.cleanup(in); + } + } + private static Object[] parse(byte[] data) { if (data == null || data.length < 4) { throw new IllegalArgumentException("empty glTF data"); @@ -182,7 +249,7 @@ private static byte[] parseGlb(byte[] data, String[] jsonHolder) { return bin; } - private static Mesh build(GraphicsDevice device, Map root, byte[] binChunk) { + private static Mesh build(Map root, byte[] binChunk) { List meshes = (List) root.get("meshes"); if (meshes == null || meshes.isEmpty()) { throw new IllegalArgumentException("glTF has no meshes"); @@ -246,9 +313,9 @@ private static Mesh build(GraphicsDevice device, Map root, byte[] binChunk) { } } - VertexBuffer vb = device.createVertexBuffer(VertexFormat.POSITION_NORMAL_TEXCOORD, vertexCount); + VertexBuffer vb = new VertexBuffer(VertexFormat.POSITION_NORMAL_TEXCOORD, vertexCount); vb.setData(interleaved); - IndexBuffer ib = device.createIndexBuffer(indices.length); + IndexBuffer ib = new IndexBuffer(indices.length); ib.setData(indices); return new Mesh(vb, ib, PrimitiveType.TRIANGLES); } @@ -393,6 +460,20 @@ private static int componentSize(int componentType) { /// texture. Only embedded images (a glTF `bufferView` or a `data:` URI) are /// supported; external image files are not fetched. private static Texture loadBaseColorTexture(GraphicsDevice device, Map root, byte[] binChunk) { + Image img = readBaseColorImage(root, binChunk); + if (img == null) { + return null; + } + Texture result = device.createTexture(img); + result.setFilter(Texture.Filter.LINEAR); + return result; + } + + /// Decodes the base-color image of the first primitive's material. Returns + /// null when the model carries no base-color texture. Only embedded images + /// (a glTF `bufferView` or a `data:` URI) are supported; external image + /// files are not fetched. + private static Image readBaseColorImage(Map root, byte[] binChunk) { List materials = (List) root.get("materials"); if (materials == null || materials.isEmpty()) { return null; @@ -421,10 +502,7 @@ private static Texture loadBaseColorTexture(GraphicsDevice device, Map root, byt if (imageBytes == null) { return null; } - Image img = Image.createImage(imageBytes, 0, imageBytes.length); - Texture result = device.createTexture(img); - result.setFilter(Texture.Filter.LINEAR); - return result; + return Image.createImage(imageBytes, 0, imageBytes.length); } private static byte[] readImageBytes(Map image, Map root, byte[] binChunk) { @@ -516,4 +594,27 @@ public Texture getBaseColorTexture() { return baseColorTexture; } } + + /// A loaded glTF model in device-free form: its geometry plus the decoded + /// base-color image extracted from the model's first material, if any. + /// Returned by `loadImageModel`. + public static final class GltfImageModel { + private final Mesh mesh; + private final Image baseColorImage; + + GltfImageModel(Mesh mesh, Image baseColorImage) { + this.mesh = mesh; + this.baseColorImage = baseColorImage; + } + + /// The model geometry. + public Mesh getMesh() { + return mesh; + } + + /// The decoded base-color image, or null when the model has none. + public Image getBaseColorImage() { + return baseColorImage; + } + } } diff --git a/CodenameOne/src/com/codename1/gpu/Primitives.java b/CodenameOne/src/com/codename1/gpu/Primitives.java index 4d33fb14632..de4c51a157f 100644 --- a/CodenameOne/src/com/codename1/gpu/Primitives.java +++ b/CodenameOne/src/com/codename1/gpu/Primitives.java @@ -109,4 +109,130 @@ public static Mesh quad(GraphicsDevice device, float size) { ib.setData(idx); return new Mesh(vb, ib, PrimitiveType.TRIANGLES); } + + /// Builds a UV sphere centered at the origin with equirectangular texture + /// coordinates: `u` wraps the longitude from 0 to 1 and `v` runs from 0 at + /// the north pole (+Y) to 1 at the south pole. This is the mapping used by + /// 360 degree panorama images. The seam column and the pole rows duplicate + /// vertices so texture coordinates stay continuous. For an inside-out + /// sphere the `u` direction is reversed so a panorama viewed from the + /// center reads correctly instead of mirror-imaged. + /// + /// #### Parameters + /// + /// - `device`: the device that allocates the buffers + /// + /// - `radius`: the sphere radius + /// + /// - `latBands`: the number of latitude subdivisions, at least 2 + /// + /// - `lonBands`: the number of longitude subdivisions, at least 3 + /// + /// - `insideOut`: when true the normals point toward the center and the + /// winding is flipped so the inner surface faces the viewer, as needed + /// when rendering a panorama from inside the sphere + /// + /// #### Returns + /// + /// an indexed triangle mesh + public static Mesh sphere(GraphicsDevice device, float radius, int latBands, int lonBands, + boolean insideOut) { + return sphere(radius, latBands, lonBands, insideOut); + } + + /// Builds a UV sphere without a `GraphicsDevice`, allocating the buffers + /// directly. Behaves exactly like + /// `sphere(GraphicsDevice, float, int, int, boolean)`; useful for preparing + /// geometry off the render thread or handing meshes to non GPU consumers + /// such as the AR content pipeline. + /// + /// #### Parameters + /// + /// - `radius`: the sphere radius + /// + /// - `latBands`: the number of latitude subdivisions, at least 2 + /// + /// - `lonBands`: the number of longitude subdivisions, at least 3 + /// + /// - `insideOut`: when true the normals point toward the center and the + /// winding is flipped + /// + /// #### Returns + /// + /// an indexed triangle mesh + public static Mesh sphere(float radius, int latBands, int lonBands, boolean insideOut) { + if (radius <= 0.0f) { + throw new IllegalArgumentException("radius must be positive"); + } + if (latBands < 2) { + throw new IllegalArgumentException("latBands must be at least 2"); + } + if (lonBands < 3) { + throw new IllegalArgumentException("lonBands must be at least 3"); + } + int vertexCount = (latBands + 1) * (lonBands + 1); + if (vertexCount > 65536) { + throw new IllegalArgumentException("sphere tessellation exceeds the 16 bit index range"); + } + float[] v = new float[vertexCount * 8]; + int o = 0; + for (int lat = 0; lat <= latBands; lat++) { + double theta = Math.PI * lat / latBands; + float sinTheta = (float) Math.sin(theta); + float cosTheta = (float) Math.cos(theta); + for (int lon = 0; lon <= lonBands; lon++) { + double phi = 2.0 * Math.PI * lon / lonBands; + float nx = sinTheta * (float) Math.sin(phi); + float ny = cosTheta; + float nz = sinTheta * (float) Math.cos(phi); + v[o] = nx * radius; + v[o + 1] = ny * radius; + v[o + 2] = nz * radius; + if (insideOut) { + v[o + 3] = -nx; + v[o + 4] = -ny; + v[o + 5] = -nz; + } else { + v[o + 3] = nx; + v[o + 4] = ny; + v[o + 5] = nz; + } + // Looking outward from inside the sphere reverses the + // apparent horizontal direction, so flip u for inside-out + // spheres to keep panorama content un-mirrored. + v[o + 6] = insideOut ? 1f - (float) lon / lonBands : (float) lon / lonBands; + v[o + 7] = (float) lat / latBands; + o += 8; + } + } + int[] idx = new int[latBands * lonBands * 6]; + int i = 0; + for (int lat = 0; lat < latBands; lat++) { + for (int lon = 0; lon < lonBands; lon++) { + int first = lat * (lonBands + 1) + lon; + int second = first + lonBands + 1; + if (insideOut) { + idx[i] = first; + idx[i + 1] = first + 1; + idx[i + 2] = second; + idx[i + 3] = second; + idx[i + 4] = first + 1; + idx[i + 5] = second + 1; + } else { + idx[i] = first; + idx[i + 1] = second; + idx[i + 2] = first + 1; + idx[i + 3] = second; + idx[i + 4] = second + 1; + idx[i + 5] = first + 1; + } + i += 6; + } + } + VertexBuffer vb = new VertexBuffer(VertexFormat.POSITION_NORMAL_TEXCOORD, vertexCount); + vb.setData(v); + IndexBuffer ib = new IndexBuffer(idx.length); + ib.setData(idx); + return new Mesh(vb, ib, PrimitiveType.TRIANGLES); + } } diff --git a/CodenameOne/src/com/codename1/gpu/Quaternion.java b/CodenameOne/src/com/codename1/gpu/Quaternion.java new file mode 100644 index 00000000000..692635c2148 --- /dev/null +++ b/CodenameOne/src/com/codename1/gpu/Quaternion.java @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codename1.gpu; + +import com.codename1.util.MathUtil; + +/// Portable quaternion math used by the 3D, AR and VR APIs. Every operation +/// works on plain `float[4]` arrays laid out as `{x, y, z, w}` so it behaves +/// identically on every platform. A quaternion of this form represents a +/// rotation; `{0, 0, 0, 1}` is the identity (no rotation). Rotation matrices +/// produced by `toMatrix(float[], float[])` use the same column-major layout +/// as `Matrix4`. +public final class Quaternion { + private Quaternion() { + } + + /// Allocates a new identity quaternion `{0, 0, 0, 1}`. + public static float[] identity() { + return new float[]{0.0f, 0.0f, 0.0f, 1.0f}; + } + + /// Resets the supplied quaternion to the identity rotation. + public static void setIdentity(float[] q) { + q[0] = 0.0f; + q[1] = 0.0f; + q[2] = 0.0f; + q[3] = 1.0f; + } + + /// Copies the contents of `src` into `dst`. Both arrays must hold 4 floats. + public static void copy(float[] src, float[] dst) { + dst[0] = src[0]; + dst[1] = src[1]; + dst[2] = src[2]; + dst[3] = src[3]; + } + + /// Multiplies `a * b` (apply `b` first, then `a`) and stores the result in + /// `dst`. `dst` may alias `a` or `b`. + public static void multiply(float[] a, float[] b, float[] dst) { + float ax = a[0]; + float ay = a[1]; + float az = a[2]; + float aw = a[3]; + float bx = b[0]; + float by = b[1]; + float bz = b[2]; + float bw = b[3]; + dst[0] = aw * bx + ax * bw + ay * bz - az * by; + dst[1] = aw * by - ax * bz + ay * bw + az * bx; + dst[2] = aw * bz + ax * by - ay * bx + az * bw; + dst[3] = aw * bw - ax * bx - ay * by - az * bz; + } + + /// Returns a quaternion representing a rotation of `angleRadians` around the + /// axis `(x, y, z)`. The axis need not be normalized; a zero axis returns + /// the identity. + public static float[] fromAxisAngle(float angleRadians, float x, float y, float z) { + float[] q = identity(); + setAxisAngle(q, angleRadians, x, y, z); + return q; + } + + /// Stores a rotation of `angleRadians` around the axis `(x, y, z)` into `q`. + /// The axis need not be normalized; a zero axis produces the identity. + public static void setAxisAngle(float[] q, float angleRadians, float x, float y, float z) { + float len = (float) Math.sqrt(x * x + y * y + z * z); + if (len == 0.0f) { + setIdentity(q); + return; + } + float half = angleRadians * 0.5f; + float s = (float) Math.sin(half) / len; + q[0] = x * s; + q[1] = y * s; + q[2] = z * s; + q[3] = (float) Math.cos(half); + } + + /// Normalizes `q` in place to unit length. A zero quaternion is reset to the + /// identity. + public static void normalize(float[] q) { + float len = (float) Math.sqrt(q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]); + if (len == 0.0f) { + setIdentity(q); + return; + } + float inv = 1.0f / len; + q[0] *= inv; + q[1] *= inv; + q[2] *= inv; + q[3] *= inv; + } + + /// Stores the conjugate of `q` (the inverse rotation for a unit quaternion) + /// into `dst`. `dst` may alias `q`. + public static void conjugate(float[] q, float[] dst) { + dst[0] = -q[0]; + dst[1] = -q[1]; + dst[2] = -q[2]; + dst[3] = q[3]; + } + + /// Writes the rotation matrix equivalent of the unit quaternion `q` into the + /// 16 element column-major matrix `dst16`. The result matches + /// `Matrix4.rotation(float, float, float, float)` for the same axis and + /// angle. + public static void toMatrix(float[] q, float[] dst16) { + float x = q[0]; + float y = q[1]; + float z = q[2]; + float w = q[3]; + float x2 = x + x; + float y2 = y + y; + float z2 = z + z; + float xx = x * x2; + float yy = y * y2; + float zz = z * z2; + float xy = x * y2; + float xz = x * z2; + float yz = y * z2; + float wx = w * x2; + float wy = w * y2; + float wz = w * z2; + dst16[0] = 1.0f - (yy + zz); + dst16[1] = xy + wz; + dst16[2] = xz - wy; + dst16[3] = 0.0f; + dst16[4] = xy - wz; + dst16[5] = 1.0f - (xx + zz); + dst16[6] = yz + wx; + dst16[7] = 0.0f; + dst16[8] = xz + wy; + dst16[9] = yz - wx; + dst16[10] = 1.0f - (xx + yy); + dst16[11] = 0.0f; + dst16[12] = 0.0f; + dst16[13] = 0.0f; + dst16[14] = 0.0f; + dst16[15] = 1.0f; + } + + /// Rotates the vector stored in `xyzInOut` (3 floats) by the unit quaternion + /// `q`, writing the result back in place. + public static void rotateVector(float[] q, float[] xyzInOut) { + float vx = xyzInOut[0]; + float vy = xyzInOut[1]; + float vz = xyzInOut[2]; + float qx = q[0]; + float qy = q[1]; + float qz = q[2]; + float qw = q[3]; + // t = 2 * cross(q.xyz, v); v' = v + qw * t + cross(q.xyz, t) + float tx = 2.0f * (qy * vz - qz * vy); + float ty = 2.0f * (qz * vx - qx * vz); + float tz = 2.0f * (qx * vy - qy * vx); + xyzInOut[0] = vx + qw * tx + (qy * tz - qz * ty); + xyzInOut[1] = vy + qw * ty + (qz * tx - qx * tz); + xyzInOut[2] = vz + qw * tz + (qx * ty - qy * tx); + } + + /// Spherically interpolates between the unit quaternions `a` and `b` by the + /// factor `t` in `[0, 1]`, storing the result in `dst`. Takes the shortest + /// arc; falls back to linear interpolation when the quaternions are nearly + /// parallel. + public static void slerp(float[] a, float[] b, float t, float[] dst) { + float ax = a[0]; + float ay = a[1]; + float az = a[2]; + float aw = a[3]; + float bx = b[0]; + float by = b[1]; + float bz = b[2]; + float bw = b[3]; + float dot = ax * bx + ay * by + az * bz + aw * bw; + if (dot < 0.0f) { + dot = -dot; + bx = -bx; + by = -by; + bz = -bz; + bw = -bw; + } + float wa; + float wb; + if (dot > 0.9995f) { + wa = 1.0f - t; + wb = t; + } else { + float theta = (float) MathUtil.acos(dot); + float invSin = 1.0f / (float) Math.sin(theta); + wa = (float) Math.sin((1.0f - t) * theta) * invSin; + wb = (float) Math.sin(t * theta) * invSin; + } + dst[0] = wa * ax + wb * bx; + dst[1] = wa * ay + wb * by; + dst[2] = wa * az + wb * bz; + dst[3] = wa * aw + wb * bw; + normalize(dst); + } + + /// Integrates a body-frame angular velocity into the orientation quaternion + /// `q`, storing the result in `dst`. `gx`, `gy` and `gz` are rotation rates + /// in radians per second around the body X, Y and Z axes (the convention + /// used by gyroscope sensors) and `dtSeconds` is the integration interval. + /// `dst` may alias `q`. The result is normalized. + public static void integrateGyro(float[] q, float gx, float gy, float gz, + float dtSeconds, float[] dst) { + float angle = (float) Math.sqrt(gx * gx + gy * gy + gz * gz) * dtSeconds; + if (angle == 0.0f) { + copy(q, dst); + return; + } + float[] delta = fromAxisAngle(angle, gx, gy, gz); + // Body-frame rates compose on the right of the current orientation. + multiply(q, delta, dst); + normalize(dst); + } +} diff --git a/CodenameOne/src/com/codename1/impl/ARImpl.java b/CodenameOne/src/com/codename1/impl/ARImpl.java new file mode 100644 index 00000000000..e5ded918442 --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/ARImpl.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl; + +import com.codename1.ar.ARAnchor; +import com.codename1.ar.ARCapabilities; +import com.codename1.ar.ARHitResult; +import com.codename1.ar.ARLightEstimate; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARPose; +import com.codename1.ar.ARSessionOptions; +import com.codename1.ar.ARTrackingFailureReason; +import com.codename1.ar.ARTrackingState; +import com.codename1.ui.PeerComponent; +import com.codename1.util.AsyncResource; + +import java.io.IOException; + +/// Per-session platform contract behind `com.codename1.ar.ARSession`. +/// +/// **Not part of the public API.** Each port subclasses this with a concrete +/// implementation; `CodenameOneImplementation#createARImpl()` is the factory +/// and returns null on platforms without AR support. One `ARImpl` instance +/// backs exactly one `ARSession`. +/// +/// The core installs an `EventSink` before calling +/// `#open(ARSessionOptions)`. Implementations may invoke the sink from any +/// thread; the core marshals events to the EDT, coalescing high-frequency +/// updates. +/// +/// @hidden +public abstract class ARImpl { + + /// Receives platform AR events. Installed by the core session before + /// `ARImpl#open(ARSessionOptions)`; implementations may call it from any + /// thread. + public interface EventSink { + /// The session's overall tracking quality changed. + void onTrackingStateChanged(ARTrackingState state, ARTrackingFailureReason reason); + + /// A new plane was detected. The implementation constructs the + /// `ARPlane` snapshot. + void onPlaneAdded(ARPlane plane); + + /// A known plane was refined. `plane` is a new snapshot carrying the + /// same id. + void onPlaneUpdated(ARPlane plane); + + /// A plane was removed, for example merged into another plane. + void onPlaneRemoved(String planeId); + + /// An anchor appeared - either confirmed by the platform after + /// `ARImpl#createAnchor(ARPose)` for ids the core does not know yet, + /// or recognized spontaneously (construct `ARImageAnchor` / + /// `ARFaceAnchor` subtypes for images and faces). + void onAnchorAdded(ARAnchor anchor); + + /// An anchor's pose or tracking state was refined. + void onAnchorUpdated(String anchorId, ARPose pose, ARTrackingState state); + + /// A face anchor's geometry was refined. Arrays may be null when the + /// platform does not supply that data; `regionPoses` is indexed by + /// `com.codename1.ar.ARFaceRegion` ordinal. + void onFaceAnchorUpdated(String anchorId, ARPose pose, ARTrackingState state, + ARPose[] regionPoses, float[] meshVertices, + int[] meshTriangles); + + /// An anchor was removed by the platform, for example a tracked image + /// or face that left the camera view permanently. + void onAnchorRemoved(String anchorId); + + /// A new light estimate is available. The core caches the latest + /// value for polling; no per-frame events reach the application. + void onLightEstimate(ARLightEstimate estimate); + + /// A new device pose is available. The core caches the latest value + /// for polling; no per-frame events reach the application. + void onCameraPose(ARPose pose); + } + + /// Returns what this device supports. Callable before + /// `#open(ARSessionOptions)`; used by `com.codename1.ar.AR#getCapabilities()` + /// to probe without starting a session. + public abstract ARCapabilities getCapabilities(); + + /// Installs the event sink. Called exactly once, before + /// `#open(ARSessionOptions)`. + public abstract void setEventSink(EventSink sink); + + /// Starts the platform AR session with the supplied configuration. Throws + /// `IOException` when the session cannot start, for example when the + /// camera permission is denied or the mode is unsupported. + public abstract void open(ARSessionOptions opts) throws IOException; + + /// Creates the component that renders the camera image composited with + /// the anchored content. Called at most once per session. + public abstract PeerComponent createViewPeer(); + + /// Performs a hit test from the normalized view coordinate (`0.0` top + /// left to `1.0` bottom right) into the world. Resolve `result` on the + /// EDT with the intersections ordered nearest first; resolve with an + /// empty array when nothing was hit. + public abstract void hitTest(float xNorm, float yNorm, AsyncResource result); + + /// Creates a world anchor at the supplied pose and returns its stable id. + public abstract String createAnchor(ARPose pose); + + /// Creates an anchor from a hit result, letting the platform anchor to + /// the exact native raycast when `nativeHandle` is non-null; otherwise + /// behaves like `#createAnchor(ARPose)`. + public abstract String createAnchorFromHit(Object nativeHandle, ARPose pose); + + /// Removes the anchor (and any attached content) from the platform + /// session. + public abstract void removeAnchor(String anchorId); + + /// Attaches, replaces or removes (`node` is null) the content root + /// rendered at the anchor. + public abstract void setAnchorNode(String anchorId, ARNode node); + + /// The attached node subtree was mutated (transform, visibility or + /// children); re-sync the platform renderer. + public abstract void nodeChanged(String anchorId, ARNode node); + + /// Requests the camera permission needed for AR. Resolve `result` on the + /// EDT with true when granted. + public abstract void requestPermissions(AsyncResource result); + + /// Suspends tracking and the camera while keeping the session object + /// usable. Pair with `#resume()`. + public abstract void pause(); + + /// Re-acquires the camera and resumes tracking after `#pause()`. + public abstract void resume(); + + /// Releases all native resources for this session. Subsequent calls + /// become no-ops. + public abstract void close(); +} diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index a2f36ab2f1d..af005ec62fe 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -7216,6 +7216,15 @@ public CameraImpl createCameraImpl() { return null; } + /// Factory for the `com.codename1.ar.AR` augmented reality API. Each call + /// returns a fresh per-session backend, or `null` on platforms without AR + /// support. Subclasses override to wire in their port. + /// + /// @hidden + public ARImpl createARImpl() { + return null; + } + /// Captures a screenshot of the screen. /// /// #### Returns diff --git a/CodenameOne/src/com/codename1/ui/Display.java b/CodenameOne/src/com/codename1/ui/Display.java index e0d93d6cf8a..28d00b085e2 100644 --- a/CodenameOne/src/com/codename1/ui/Display.java +++ b/CodenameOne/src/com/codename1/ui/Display.java @@ -6083,6 +6083,16 @@ public com.codename1.impl.CameraImpl getCameraBackend() { return impl.createCameraImpl(); } + /// Creates a fresh per-session backend for the `com.codename1.ar.AR` + /// augmented reality API. Returns `null` on platforms without AR support. + /// Application code should use `AR.open(...)` rather than calling this + /// directly. + /// + /// @hidden + public com.codename1.impl.ARImpl getARBackend() { + return impl.createARImpl(); + } + /// Indicates whether the native picker dialog is supported for the given type /// which can include one of PICKER_TYPE_DATE_AND_TIME, PICKER_TYPE_TIME, PICKER_TYPE_DATE /// diff --git a/CodenameOne/src/com/codename1/vr/HeadTracker.java b/CodenameOne/src/com/codename1/vr/HeadTracker.java new file mode 100644 index 00000000000..007dca12d69 --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/HeadTracker.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Quaternion; +import com.codename1.sensors.MotionEvent; +import com.codename1.sensors.MotionSensor; +import com.codename1.sensors.MotionSensorListener; +import com.codename1.sensors.MotionSensorManager; + +/// Tracks the device orientation for VR by feeding the motion sensors +/// (gyroscope, accelerometer and magnetometer through +/// `com.codename1.sensors.MotionSensorManager`) into an `OrientationFilter`. +/// +/// The gyroscope drives the orientation; without one the tracker falls back +/// to accelerometer tilt (stable pitch and roll, no reliable yaw). Sensor +/// hardware runs only between `#start()` and `#stop()` - the sensors API +/// reference-counts listeners, so a stopped tracker costs no battery. +/// +/// `#getOrientation(float[])` is thread safe: sensor events arrive on the +/// EDT while the render thread reads the latest snapshot. +public final class HeadTracker { + private final MotionSensorManager manager; + private final OrientationFilter filter = new OrientationFilter(); + private final Object lock = new Object(); + private final float[] snapshot = Quaternion.identity(); + + private MotionSensor gyro; + private MotionSensor accel; + private MotionSensor mag; + private MotionSensorListener gyroListener; + private MotionSensorListener accelListener; + private MotionSensorListener magListener; + + private long lastTimestamp; + private float ax; + private float ay; + private float az; + private boolean hasAccel; + private float mx = Float.NaN; + private float my = Float.NaN; + private float mz = Float.NaN; + private boolean started; + + /// Creates a tracker over the platform motion sensors. + public HeadTracker() { + this(MotionSensorManager.getInstance()); + } + + /// Test hook: creates a tracker over the supplied manager. + HeadTracker(MotionSensorManager manager) { + this.manager = manager; + } + + /// True when the device has the sensors head tracking needs: a gyroscope, + /// or at least an accelerometer for tilt-only tracking. + public static boolean isSupported() { + MotionSensorManager m = MotionSensorManager.getInstance(); + return m.isSensorSupported(MotionSensorManager.TYPE_GYROSCOPE) + || m.isSensorSupported(MotionSensorManager.TYPE_ACCELEROMETER); + } + + /// The fusion filter behind this tracker, exposed to tune the + /// gyro/reference blend. Configure before `#start()`. + public OrientationFilter getFilter() { + return filter; + } + + /// Starts the sensors and orientation updates. Idempotent. + public void start() { + if (started) { + return; + } + started = true; + lastTimestamp = 0; + boolean hasGyro = manager.isSensorSupported(MotionSensorManager.TYPE_GYROSCOPE); + if (hasGyro) { + gyro = manager.getSensor(MotionSensorManager.TYPE_GYROSCOPE); + gyroListener = new MotionSensorListener() { + @Override public void motionReceived(MotionEvent evt) { + step(evt.getX(), evt.getY(), evt.getZ(), evt.getTimestamp()); + } + }; + gyro.addListener(gyroListener); + } + if (manager.isSensorSupported(MotionSensorManager.TYPE_ACCELEROMETER)) { + accel = manager.getSensor(MotionSensorManager.TYPE_ACCELEROMETER); + final boolean drivesFilter = !hasGyro; + accelListener = new MotionSensorListener() { + @Override public void motionReceived(MotionEvent evt) { + synchronized (lock) { + ax = evt.getX(); + ay = evt.getY(); + az = evt.getZ(); + hasAccel = true; + } + if (drivesFilter) { + // No gyroscope: the accelerometer cadence drives the + // filter with zero rotation rates, converging tilt. + step(0f, 0f, 0f, evt.getTimestamp()); + } + } + }; + accel.addListener(accelListener); + } + if (manager.isSensorSupported(MotionSensorManager.TYPE_MAGNETOMETER)) { + mag = manager.getSensor(MotionSensorManager.TYPE_MAGNETOMETER); + magListener = new MotionSensorListener() { + @Override public void motionReceived(MotionEvent evt) { + synchronized (lock) { + mx = evt.getX(); + my = evt.getY(); + mz = evt.getZ(); + } + } + }; + mag.addListener(magListener); + } + } + + /// Stops the sensors. Idempotent; the last orientation remains readable. + public void stop() { + if (!started) { + return; + } + started = false; + if (gyro != null && gyroListener != null) { + gyro.removeListener(gyroListener); + } + if (accel != null && accelListener != null) { + accel.removeListener(accelListener); + } + if (mag != null && magListener != null) { + mag.removeListener(magListener); + } + gyroListener = null; + accelListener = null; + magListener = null; + } + + /// True between `#start()` and `#stop()`. + public boolean isStarted() { + return started; + } + + /// Rotates the orientation so the current view direction becomes + /// "straight ahead", keeping pitch and roll. + public void recenter() { + synchronized (lock) { + filter.recenterYaw(); + filter.getOrientation(snapshot); + } + } + + /// Copies the latest orientation quaternion into `quatOut4` as + /// `{x, y, z, w}`. Thread safe; intended to be read from the render + /// thread. + public void getOrientation(float[] quatOut4) { + synchronized (lock) { + Quaternion.copy(snapshot, quatOut4); + } + } + + private void step(float gx, float gy, float gz, long timestampMillis) { + synchronized (lock) { + if (lastTimestamp != 0 && timestampMillis > lastTimestamp) { + float dt = (timestampMillis - lastTimestamp) / 1000f; + float sax = hasAccel ? ax : 0f; + float say = hasAccel ? ay : 0f; + float saz = hasAccel ? az : 0f; + filter.update(gx, gy, gz, sax, say, saz, mx, my, mz, dt); + filter.getOrientation(snapshot); + } + lastTimestamp = timestampMillis; + } + } +} diff --git a/CodenameOne/src/com/codename1/vr/Media360View.java b/CodenameOne/src/com/codename1/vr/Media360View.java new file mode 100644 index 00000000000..7090c70317e --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/Media360View.java @@ -0,0 +1,421 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Material; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Primitives; +import com.codename1.gpu.Quaternion; +import com.codename1.gpu.RenderView; +import com.codename1.gpu.Renderer; +import com.codename1.gpu.Texture; +import com.codename1.ui.Container; +import com.codename1.ui.Display; +import com.codename1.ui.Image; +import com.codename1.ui.layouts.BorderLayout; + +/// A 360 degree panorama viewer: renders an equirectangular image onto the +/// inside of a sphere with drag-to-look navigation and optional gyroscope +/// look. Works everywhere `Display#isGpuSupported()` is true; on platforms +/// without a GPU backend it degrades to the standard "3D not supported" +/// placeholder without crashing. +/// +/// Mono mode fills the component; `#setStereo(boolean)` renders the same +/// viewpoint side by side for cardboard-style viewers. Photo spheres are +/// captured from a single point, so stereo intentionally uses a zero eye +/// separation - there is no parallax information in the image to reproduce. +/// +/// 360 video is not supported directly because the platform has no path from +/// media frames to GPU textures yet; `#setTextureSource(TextureSource)` is +/// the extension point for dynamic content. +/// +/// ```java +/// Media360View pano = new Media360View(); +/// pano.setImage(EncodedImage.create("/panorama.jpg")); +/// form.add(BorderLayout.CENTER, pano); +/// ``` +public class Media360View extends Container { + private static final float MAX_PITCH_DEGREES = 89f; + + private final RenderView renderView; + // Guards the state the EDT writes and the render thread reads each frame + // (the codebase convention is locking over volatile fields). + private final Object stateLock = new Object(); + private Image pendingImage; + private boolean imageDirty; + private TextureSource textureSource; + private boolean sourceDirty; + private float yaw; + private float pitch; + private boolean stereo; + private boolean headTrackingEnabled; + private HeadTracker tracker; + + private int lastDragX = -1; + private int lastDragY = -1; + + /// Creates an empty viewer; supply content with `#setImage(Image)` or + /// `#setTextureSource(TextureSource)`. + public Media360View() { + super(new BorderLayout()); + renderView = new RenderView(new SphereLoop()); + add(BorderLayout.CENTER, renderView); + setFocusable(true); + } + + /// True when the current platform provides a 3D backend. + public boolean isSupported() { + return Display.getInstance().isGpuSupported(); + } + + /// Shows an equirectangular panorama image (the format produced by 360 + /// cameras and phone panorama modes: longitude maps to X, latitude to Y). + /// Replaces any previous image or texture source. + /// + /// #### Parameters + /// + /// - `image`: the panorama to display + public void setImage(Image image) { + synchronized (stateLock) { + pendingImage = image; + imageDirty = true; + } + renderView.requestRender(); + } + + /// Installs a dynamic texture source, replacing any static image. The + /// extension point for procedural or (future) video content. + /// + /// #### Parameters + /// + /// - `source`: the source, or null to remove it + public void setTextureSource(TextureSource source) { + synchronized (stateLock) { + textureSource = source; + sourceDirty = true; + } + renderView.requestRender(); + } + + /// Switches between a full-viewport mono view (the default) and + /// side-by-side stereo for cardboard-style viewers. + public void setStereo(boolean stereo) { + synchronized (stateLock) { + this.stereo = stereo; + } + renderView.requestRender(); + } + + /// True when rendering side-by-side stereo. + public boolean isStereo() { + synchronized (stateLock) { + return stereo; + } + } + + /// Enables gyroscope look-around, composed with drag navigation. When the + /// device has no motion sensors this quietly stays drag-only. + public void setHeadTrackingEnabled(boolean enabled) { + synchronized (stateLock) { + headTrackingEnabled = enabled; + } + if (enabled) { + if (tracker == null) { + tracker = new HeadTracker(); + } + if (isInitialized()) { + tracker.start(); + } + renderView.setContinuous(true); + } else { + if (tracker != null) { + tracker.stop(); + } + renderView.setContinuous(false); + } + renderView.requestRender(); + } + + /// True when gyroscope look-around is enabled. + public boolean isHeadTrackingEnabled() { + synchronized (stateLock) { + return headTrackingEnabled; + } + } + + /// The horizontal look angle in degrees; positive turns right. + public float getYaw() { + synchronized (stateLock) { + return yaw; + } + } + + /// Sets the horizontal look angle in degrees. + public void setYaw(float yawDegrees) { + synchronized (stateLock) { + this.yaw = yawDegrees; + } + renderView.requestRender(); + } + + /// The vertical look angle in degrees, clamped so the view cannot flip + /// over the poles; positive looks up. + public float getPitch() { + synchronized (stateLock) { + return pitch; + } + } + + /// Sets the vertical look angle in degrees. Clamped to stay off the + /// poles. + public void setPitch(float pitchDegrees) { + synchronized (stateLock) { + this.pitch = clampPitch(pitchDegrees); + } + renderView.requestRender(); + } + + /// Resets the view to look straight ahead and recenters the gyroscope. + public void reset() { + synchronized (stateLock) { + yaw = 0f; + pitch = 0f; + } + if (tracker != null) { + tracker.recenter(); + } + renderView.requestRender(); + } + + /// The underlying render view, exposed for advanced integration. + public RenderView getRenderView() { + return renderView; + } + + private static float clampPitch(float p) { + if (p > MAX_PITCH_DEGREES) { + return MAX_PITCH_DEGREES; + } + if (p < -MAX_PITCH_DEGREES) { + return -MAX_PITCH_DEGREES; + } + return p; + } + + @Override + protected void initComponent() { + super.initComponent(); + if (isHeadTrackingEnabled() && tracker != null) { + tracker.start(); + } + } + + @Override + protected void deinitialize() { + if (tracker != null) { + tracker.stop(); + } + super.deinitialize(); + } + + @Override + public void pointerPressed(int x, int y) { + super.pointerPressed(x, y); + lastDragX = x; + lastDragY = y; + } + + @Override + public void pointerDragged(int x, int y) { + super.pointerDragged(x, y); + if (lastDragX >= 0) { + int w = Math.max(1, getWidth()); + int h = Math.max(1, getHeight()); + // Dragging the image right turns the view left, matching the + // grab-the-world gesture users expect from panorama viewers. + synchronized (stateLock) { + yaw -= (x - lastDragX) * 180f / w; + pitch = clampPitch(pitch + (y - lastDragY) * 180f / h); + } + } + lastDragX = x; + lastDragY = y; + renderView.requestRender(); + } + + @Override + public void pointerReleased(int x, int y) { + super.pointerReleased(x, y); + lastDragX = -1; + lastDragY = -1; + } + + /// The internal renderer: an inside-out UV sphere carrying the panorama + /// texture, looked at from the origin. + private final class SphereLoop implements Renderer { + private Mesh sphere; + private Material material; + private Texture texture; + private final Camera camera = new Camera(); + private final float[] quat = Quaternion.identity(); + private final float[] dir = new float[3]; + private final float[] up = new float[3]; + private int surfaceWidth = 1; + private int surfaceHeight = 1; + + @Override + public void onInit(GraphicsDevice device) { + sphere = Primitives.sphere(device, 50f, 48, 96, true); + material = new Material(Material.Type.UNLIT).setColor(0xffffffff); + } + + @Override + public void onResize(GraphicsDevice device, int width, int height) { + surfaceWidth = Math.max(1, width); + surfaceHeight = Math.max(1, height); + } + + @Override + public void onFrame(GraphicsDevice device) { + // Snapshot the EDT-owned state once per frame. + TextureSource sourceNow; + boolean sourceDirtyNow; + Image imageNow; + boolean imageDirtyNow; + float yawNow; + float pitchNow; + boolean stereoNow; + boolean trackingNow; + synchronized (stateLock) { + sourceNow = textureSource; + sourceDirtyNow = sourceDirty; + sourceDirty = false; + imageNow = pendingImage; + imageDirtyNow = imageDirty; + imageDirty = false; + yawNow = yaw; + pitchNow = pitch; + stereoNow = stereo; + trackingNow = headTrackingEnabled; + } + refreshTexture(device, sourceNow, sourceDirtyNow, imageNow, imageDirtyNow); + computeLook(yawNow, pitchNow, trackingNow); + int w = surfaceWidth; + int h = surfaceHeight; + device.clear(0xff000000, true, true); + if (stereoNow) { + int half = w / 2; + drawSphere(device, 0, 0, half, h); + drawSphere(device, half, 0, w - half, h); + device.setViewport(0, 0, w, h); + } else { + drawSphere(device, 0, 0, w, h); + } + } + + private void refreshTexture(GraphicsDevice device, TextureSource source, + boolean sourceChanged, Image img, boolean imageChanged) { + if (sourceChanged) { + if (texture != null) { + device.dispose(texture); + texture = null; + } + } + if (source != null) { + if (texture == null) { + texture = source.createTexture(device); + if (texture != null) { + texture.setFilter(Texture.Filter.LINEAR); + } + } + if (texture != null) { + source.updateTexture(device, texture); + } + } else if (imageChanged) { + if (texture != null) { + device.dispose(texture); + texture = null; + } + if (img != null) { + texture = device.createTexture(img); + texture.setFilter(Texture.Filter.LINEAR); + } + } + material.setTexture(texture); + } + + private void computeLook(float yawDegrees, float pitchDegrees, boolean tracking) { + float yawRad = (float) Math.toRadians(yawDegrees); + float pitchRad = (float) Math.toRadians(pitchDegrees); + if (tracking && tracker != null) { + // The drag yaw acts as a manual offset on top of the gyro + // orientation; drag pitch is ignored while the gyro owns it. + tracker.getOrientation(quat); + float[] yawQ = Quaternion.fromAxisAngle(-yawRad, 0f, 1f, 0f); + Quaternion.multiply(yawQ, quat, quat); + dir[0] = 0f; + dir[1] = 0f; + dir[2] = -1f; + Quaternion.rotateVector(quat, dir); + up[0] = 0f; + up[1] = 1f; + up[2] = 0f; + Quaternion.rotateVector(quat, up); + } else { + float cp = (float) Math.cos(pitchRad); + dir[0] = cp * (float) Math.sin(yawRad); + dir[1] = (float) Math.sin(pitchRad); + dir[2] = -cp * (float) Math.cos(yawRad); + up[0] = 0f; + up[1] = 1f; + up[2] = 0f; + } + camera.setPosition(0f, 0f, 0f); + camera.setTarget(dir[0], dir[1], dir[2]); + camera.setUp(up[0], up[1], up[2]); + camera.setPerspective(75f, 0.1f, 100f); + } + + private void drawSphere(GraphicsDevice device, int x, int y, int w, int h) { + device.setViewport(x, y, w, h); + camera.setAspect((float) w / (float) Math.max(1, h)); + device.setCamera(camera); + device.draw(sphere, material, null); + } + + @Override + public void onDispose(GraphicsDevice device) { + TextureSource source = textureSource; + if (source != null && device != null) { + source.dispose(device); + } + if (texture != null && device != null) { + device.dispose(texture); + } + texture = null; + sphere = null; + } + } +} diff --git a/CodenameOne/src/com/codename1/vr/OrientationFilter.java b/CodenameOne/src/com/codename1/vr/OrientationFilter.java new file mode 100644 index 00000000000..0fcd1372b2b --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/OrientationFilter.java @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Quaternion; +import com.codename1.util.MathUtil; + +/// Deterministic complementary sensor-fusion filter that turns raw gyroscope, +/// accelerometer and magnetometer readings into a device orientation +/// quaternion. Pure math with no platform dependencies, so the same inputs +/// always produce the same orientation on every platform. +/// +/// The gyroscope drives the orientation between samples; the accelerometer +/// slowly corrects accumulated tilt drift toward gravity and the magnetometer +/// (when available) corrects yaw drift toward magnetic north. The blend is +/// controlled by `#setGyroWeight(float)`. +/// +/// Conventions: the device frame has X right, Y up and Z toward the user +/// (the `com.codename1.sensors` convention); the world frame is Y up with -Z +/// forward. The output quaternion rotates device-frame vectors into the world +/// frame. +public final class OrientationFilter { + private final float[] q = Quaternion.identity(); + private float gyroWeight = 0.98f; + + // Scratch buffers, allocated once. + private final float[] v = new float[3]; + private final float[] correction = new float[4]; + + /// Sets the complementary blend coefficient in `(0, 1]`: the fraction of + /// each update that trusts the integrated gyroscope over the + /// accelerometer/magnetometer reference. Higher values are smoother but + /// drift-correct more slowly. Default `0.98`. + public void setGyroWeight(float w) { + if (w <= 0f || w > 1f) { + throw new IllegalArgumentException("gyroWeight must be in (0, 1]"); + } + this.gyroWeight = w; + } + + /// The complementary blend coefficient. + public float getGyroWeight() { + return gyroWeight; + } + + /// Advances the filter by one sensor sample. + /// + /// #### Parameters + /// + /// - `gx`, `gy`, `gz`: gyroscope rotation rates around the device axes in + /// radians per second; pass zeros when no gyroscope exists + /// + /// - `ax`, `ay`, `az`: accelerometer reading including gravity in meters + /// per second squared; pass zeros (or NaN) to skip tilt correction + /// + /// - `mx`, `my`, `mz`: magnetometer reading in microtesla; pass NaN to + /// skip yaw correction + /// + /// - `dtSeconds`: the time since the previous update + public void update(float gx, float gy, float gz, + float ax, float ay, float az, + float mx, float my, float mz, + float dtSeconds) { + if (dtSeconds <= 0f) { + return; + } + // 1. Gyro integration: body-frame rates compose on the right. + Quaternion.integrateGyro(q, gx, gy, gz, dtSeconds, q); + + float blend = 1f - gyroWeight; + + // 2. Tilt correction toward gravity. At rest the accelerometer + // measures the reaction to gravity along the device's world-up + // direction, so rotating the normalized reading into the world frame + // should give (0, 1, 0). + float alen = (float) Math.sqrt(ax * ax + ay * ay + az * az); + if (!isNaN(alen) && alen > 1e-6f) { + v[0] = ax / alen; + v[1] = ay / alen; + v[2] = az / alen; + Quaternion.rotateVector(q, v); + // Rotation carrying the measured up vector onto world up. + applyWorldCorrection(v[0], v[1], v[2], 0f, 1f, 0f, blend); + } + + // 3. Yaw correction toward magnetic north. Only the horizontal + // component of the field is used, and only the world-Y rotation is + // corrected so the magnetometer can never disturb pitch or roll. + if (!isNaN(mx) && !isNaN(my) && !isNaN(mz)) { + float mlen = (float) Math.sqrt(mx * mx + my * my + mz * mz); + if (mlen > 1e-6f) { + v[0] = mx / mlen; + v[1] = my / mlen; + v[2] = mz / mlen; + Quaternion.rotateVector(q, v); + float hx = v[0]; + float hz = v[2]; + float hlen = (float) Math.sqrt(hx * hx + hz * hz); + if (hlen > 1e-6f) { + // Angle of the horizontal field from world north (-Z). A + // yaw drift of epsilon shows up here as -epsilon, so + // applying the measured angle directly cancels the drift. + float yawError = (float) MathUtil.atan2(hx, -hz); + Quaternion.setAxisAngle(correction, yawError * blend, 0f, 1f, 0f); + Quaternion.multiply(correction, q, q); + Quaternion.normalize(q); + } + } + } + } + + /// Applies a world-frame rotation moving `from` a fraction of the way + /// toward `to`. + private void applyWorldCorrection(float fx, float fy, float fz, + float tx, float ty, float tz, float fraction) { + float cx = fy * tz - fz * ty; + float cy = fz * tx - fx * tz; + float cz = fx * ty - fy * tx; + float clen = (float) Math.sqrt(cx * cx + cy * cy + cz * cz); + float dot = fx * tx + fy * ty + fz * tz; + if (clen < 1e-6f) { + return; + } + float angle = (float) MathUtil.atan2(clen, dot); + Quaternion.setAxisAngle(correction, angle * fraction, cx, cy, cz); + Quaternion.multiply(correction, q, q); + Quaternion.normalize(q); + } + + private static boolean isNaN(float f) { + return f != f; + } + + /// Copies the current orientation quaternion into `quatOut4` as + /// `{x, y, z, w}`. + public void getOrientation(float[] quatOut4) { + Quaternion.copy(q, quatOut4); + } + + /// The current orientation quaternion as a newly allocated array. + public float[] getOrientation() { + float[] out = new float[4]; + getOrientation(out); + return out; + } + + /// Resets the filter to the identity orientation. + public void reset() { + Quaternion.setIdentity(q); + } + + /// Rotates the orientation around world up so the current forward + /// direction becomes the new "straight ahead", keeping pitch and roll. + /// Use to let the user recenter the view. + public void recenterYaw() { + // The device -Z axis in world space is the current forward direction. + v[0] = 0f; + v[1] = 0f; + v[2] = -1f; + Quaternion.rotateVector(q, v); + float hlen = (float) Math.sqrt(v[0] * v[0] + v[2] * v[2]); + if (hlen < 1e-6f) { + // Looking straight up or down; yaw is undefined. + return; + } + // A device yaw of theta puts the forward vector at atan2 angle + // -theta, so applying the measured angle directly cancels the yaw. + float yaw = (float) MathUtil.atan2(v[0], -v[2]); + Quaternion.setAxisAngle(correction, yaw, 0f, 1f, 0f); + Quaternion.multiply(correction, q, q); + Quaternion.normalize(q); + } +} diff --git a/CodenameOne/src/com/codename1/vr/TextureSource.java b/CodenameOne/src/com/codename1/vr/TextureSource.java new file mode 100644 index 00000000000..82a502794d3 --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/TextureSource.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Texture; + +/// Supplies and updates the texture rendered by a `Media360View`, enabling +/// dynamic content such as procedural animations - and, once a platform path +/// from media frames to GPU textures exists, 360 video. All methods run on +/// the render thread. +public interface TextureSource { + /// Creates the texture. Invoked once on the render thread after the GPU + /// context is ready. + /// + /// #### Parameters + /// + /// - `device`: the graphics device bound to the view + /// + /// #### Returns + /// + /// the texture to map onto the sphere + Texture createTexture(GraphicsDevice device); + + /// Invoked once per frame before drawing. Update the texture contents + /// here and return true when they changed (so continuous sources keep the + /// view animating). + /// + /// #### Parameters + /// + /// - `device`: the graphics device bound to the view + /// + /// - `texture`: the texture returned by `#createTexture(GraphicsDevice)` + /// + /// #### Returns + /// + /// true when the texture contents changed this frame + boolean updateTexture(GraphicsDevice device, Texture texture); + + /// Invoked when the view tears down. Release anything the source owns; + /// the texture itself is disposed by the view. + /// + /// #### Parameters + /// + /// - `device`: the graphics device bound to the view + void dispose(GraphicsDevice device); +} diff --git a/CodenameOne/src/com/codename1/vr/VRCameraRig.java b/CodenameOne/src/com/codename1/vr/VRCameraRig.java new file mode 100644 index 00000000000..26cc894974f --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/VRCameraRig.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.Quaternion; + +/// Computes per-eye cameras for stereo rendering: a head position and +/// orientation plus a `VRSettings` interpupillary distance yield a +/// `com.codename1.gpu.Camera` per `VREye`. Pure math over the existing gpu +/// camera; usable from any thread that owns the target camera. +public final class VRCameraRig { + private final VRSettings settings; + private float x; + private float y; + private float z; + private final float[] q = Quaternion.identity(); + + /// Creates a rig with the supplied settings. + /// + /// #### Parameters + /// + /// - `settings`: the eye separation and lens parameters + public VRCameraRig(VRSettings settings) { + this.settings = settings == null ? new VRSettings() : settings; + } + + /// The settings this rig was created with. + public VRSettings getSettings() { + return settings; + } + + /// Sets the head center position in world space. + public void setPosition(float x, float y, float z) { + this.x = x; + this.y = y; + this.z = z; + } + + /// Sets the head orientation as an `{x, y, z, w}` quaternion rotating + /// head-local vectors into world space. + public void setOrientation(float[] quat4) { + Quaternion.copy(quat4, q); + } + + /// Configures `out` as the camera for the supplied eye: positioned at the + /// head center offset by half the interpupillary distance along the head's + /// right vector, looking along the head's forward vector. + /// + /// #### Parameters + /// + /// - `out`: the camera to configure + /// + /// - `eye`: which eye to compute + /// + /// - `aspect`: the per-eye viewport aspect ratio (width / height) + public void apply(Camera out, VREye eye, float aspect) { + float[] v = new float[3]; + float half = settings.getIpdMeters() * 0.5f * eye.offsetSign(); + v[0] = half; + v[1] = 0f; + v[2] = 0f; + Quaternion.rotateVector(q, v); + float ex = x + v[0]; + float ey = y + v[1]; + float ez = z + v[2]; + + v[0] = 0f; + v[1] = 0f; + v[2] = -1f; + Quaternion.rotateVector(q, v); + float fx = v[0]; + float fy = v[1]; + float fz = v[2]; + + v[0] = 0f; + v[1] = 1f; + v[2] = 0f; + Quaternion.rotateVector(q, v); + + out.setPosition(ex, ey, ez); + out.setTarget(ex + fx, ey + fy, ez + fz); + out.setUp(v[0], v[1], v[2]); + out.setPerspective(settings.getFovYDegrees(), settings.getNear(), settings.getFar()); + out.setAspect(aspect); + } +} diff --git a/CodenameOne/src/com/codename1/vr/VREye.java b/CodenameOne/src/com/codename1/vr/VREye.java new file mode 100644 index 00000000000..006d39949f2 --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/VREye.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +/// Identifies which eye a `VRRenderer` frame is being rendered for. +public enum VREye { + /// The left eye in stereo mode. Offset half the interpupillary distance to + /// the left of the head center. + LEFT, + + /// The right eye in stereo mode. Offset half the interpupillary distance + /// to the right of the head center. + RIGHT, + + /// The single centered viewpoint used in mono mode. + CENTER; + + /// The sign of this eye's horizontal offset from the head center: `-1` + /// for `LEFT`, `1` for `RIGHT`, `0` for `CENTER`. + public int offsetSign() { + if (this == LEFT) { + return -1; + } + if (this == RIGHT) { + return 1; + } + return 0; + } +} diff --git a/CodenameOne/src/com/codename1/vr/VRRenderer.java b/CodenameOne/src/com/codename1/vr/VRRenderer.java new file mode 100644 index 00000000000..a5e6eed6e6a --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/VRRenderer.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.GraphicsDevice; + +/// Application supplied callback that draws the scene for a `VRView`. Like +/// `com.codename1.gpu.Renderer`, every method runs on the platform render +/// thread that owns the GPU context - never touch Codename One UI components +/// from these callbacks. +/// +/// The view clears the frame, sets the per-eye viewport and camera, then +/// invokes `#onEyeFrame(GraphicsDevice, VREye, Camera)` once per eye (twice +/// per frame in stereo, once with `VREye#CENTER` in mono). Draw the same +/// scene for every eye; the camera differences produce the stereo effect. +public interface VRRenderer { + /// Invoked once after the GPU context is created. Allocate buffers, + /// textures and materials here. + /// + /// #### Parameters + /// + /// - `device`: the graphics device bound to this view + void onInit(GraphicsDevice device); + + /// Invoked once per eye per frame to render the scene. The viewport and + /// camera are already configured for this eye. + /// + /// #### Parameters + /// + /// - `device`: the graphics device bound to this view + /// + /// - `eye`: which eye is being rendered + /// + /// - `eyeCamera`: the camera for this eye, already applied to the device + void onEyeFrame(GraphicsDevice device, VREye eye, Camera eyeCamera); + + /// Invoked when the GPU context is being torn down. Release resources not + /// owned by the device. May be invoked with a null device when the context + /// was lost. + /// + /// #### Parameters + /// + /// - `device`: the graphics device, or null if the context was lost + void onDispose(GraphicsDevice device); +} diff --git a/CodenameOne/src/com/codename1/vr/VRSettings.java b/CodenameOne/src/com/codename1/vr/VRSettings.java new file mode 100644 index 00000000000..90fab7f017b --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/VRSettings.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +/// Configuration for a `VRView`. Fluent builder. +public final class VRSettings { + private float ipdMeters = 0.064f; + private float fovYDegrees = 90f; + private float near = 0.1f; + private float far = 100f; + private boolean stereo = true; + + /// The interpupillary distance - the world-space separation between the + /// two eyes - in meters. Default `0.064` (the human average). + public VRSettings ipdMeters(float ipd) { + this.ipdMeters = ipd; + return this; + } + + /// The vertical field of view per eye in degrees. Default `90`. + public VRSettings fovYDegrees(float fov) { + this.fovYDegrees = fov; + return this; + } + + /// The near and far clip plane distances. Defaults `0.1` and `100`. + public VRSettings nearFar(float near, float far) { + this.near = near; + this.far = far; + return this; + } + + /// Whether the view renders side-by-side stereo (true, the default) or a + /// single centered viewpoint. + public VRSettings stereo(boolean stereo) { + this.stereo = stereo; + return this; + } + + /// The interpupillary distance in meters. + public float getIpdMeters() { + return ipdMeters; + } + + /// The vertical field of view per eye in degrees. + public float getFovYDegrees() { + return fovYDegrees; + } + + /// The near clip plane distance. + public float getNear() { + return near; + } + + /// The far clip plane distance. + public float getFar() { + return far; + } + + /// True when the view renders side-by-side stereo. + public boolean isStereo() { + return stereo; + } +} diff --git a/CodenameOne/src/com/codename1/vr/VRView.java b/CodenameOne/src/com/codename1/vr/VRView.java new file mode 100644 index 00000000000..658486f2659 --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/VRView.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Quaternion; +import com.codename1.gpu.RenderView; +import com.codename1.gpu.Renderer; +import com.codename1.ui.Container; +import com.codename1.ui.Display; +import com.codename1.ui.layouts.BorderLayout; + +/// Component that renders an application scene in virtual reality: stereo +/// side-by-side per-eye views with sensor-fusion head tracking, built +/// entirely on the portable `com.codename1.gpu` pipeline so it works on every +/// platform where `Display#isGpuSupported()` is true - including the +/// simulator. +/// +/// Supply a `VRRenderer` that draws the scene; the view clears the frame, +/// splits the viewport per eye, positions a `VRCameraRig` from the +/// `HeadTracker` orientation and invokes the renderer once per eye. +/// +/// IMPORTANT: `VRRenderer` callbacks run on the platform render thread, never +/// on the EDT. Do not touch Codename One UI components from them. +/// +/// ```java +/// VRView vr = new VRView(new VRRenderer() { +/// Mesh cube; +/// Material material; +/// public void onInit(GraphicsDevice device) { +/// cube = Primitives.cube(device, 0.5f); +/// material = new Material(Material.Type.PHONG).setColor(0xff3366ff); +/// } +/// public void onEyeFrame(GraphicsDevice device, VREye eye, Camera camera) { +/// device.draw(cube, material, Matrix4.translation(0, 0, -2)); +/// } +/// public void onDispose(GraphicsDevice device) { } +/// }); +/// vr.setContinuous(true); +/// form.add(BorderLayout.CENTER, vr); +/// ``` +public class VRView extends Container { + private final VRRenderer vrRenderer; + private final VRSettings settings; + private final HeadTracker tracker; + private final VRCameraRig rig; + private final RenderView renderView; + private final Camera eyeCamera = new Camera(); + private final float[] quat = Quaternion.identity(); + // Guards the settings the EDT writes and the render thread reads each + // frame (the codebase convention is locking over volatile fields). + private final Object stateLock = new Object(); + private boolean stereo; + private boolean headTrackingEnabled = true; + private int clearColor = 0xff000000; + private float posX; + private float posY; + private float posZ; + // Written and read on the render thread only. + private int surfaceWidth = 1; + private int surfaceHeight = 1; + + /// Creates a VR view with default settings. + /// + /// #### Parameters + /// + /// - `renderer`: draws the scene, once per eye per frame + public VRView(VRRenderer renderer) { + this(renderer, new VRSettings()); + } + + /// Creates a VR view. + /// + /// #### Parameters + /// + /// - `renderer`: draws the scene, once per eye per frame + /// + /// - `settings`: eye separation and lens parameters; null uses defaults + public VRView(VRRenderer renderer, VRSettings settings) { + super(new BorderLayout()); + if (renderer == null) { + throw new IllegalArgumentException("renderer is required"); + } + this.vrRenderer = renderer; + this.settings = settings == null ? new VRSettings() : settings; + this.stereo = this.settings.isStereo(); + this.tracker = new HeadTracker(); + this.rig = new VRCameraRig(this.settings); + this.renderView = new RenderView(new EyeLoop()); + add(BorderLayout.CENTER, renderView); + } + + /// True when the current platform provides a 3D backend. Equivalent to + /// `Display#isGpuSupported()`. + public boolean isSupported() { + return Display.getInstance().isGpuSupported(); + } + + /// The settings this view was created with. + public VRSettings getSettings() { + return settings; + } + + /// The head tracker driving this view. Exposed to tune the fusion filter + /// or observe the raw orientation. + public HeadTracker getHeadTracker() { + return tracker; + } + + /// Enables or disables head tracking. When disabled the last orientation + /// freezes; combine with `VRCameraRig` manually for custom control. + public void setHeadTrackingEnabled(boolean enabled) { + synchronized (stateLock) { + this.headTrackingEnabled = enabled; + } + if (isInitialized()) { + if (enabled) { + tracker.start(); + } else { + tracker.stop(); + } + } + renderView.requestRender(); + } + + /// True when head tracking drives the view orientation. + public boolean isHeadTrackingEnabled() { + synchronized (stateLock) { + return headTrackingEnabled; + } + } + + /// Rotates the view so the current direction becomes "straight ahead". + public void recenter() { + tracker.recenter(); + renderView.requestRender(); + } + + /// Switches between side-by-side stereo and a single centered viewpoint. + public void setStereo(boolean stereo) { + synchronized (stateLock) { + this.stereo = stereo; + } + renderView.requestRender(); + } + + /// True when rendering side-by-side stereo. + public boolean isStereo() { + synchronized (stateLock) { + return stereo; + } + } + + /// Sets the head center position in world space, for example to move the + /// viewer through the scene. + public void setPosition(float x, float y, float z) { + synchronized (stateLock) { + posX = x; + posY = y; + posZ = z; + } + renderView.requestRender(); + } + + /// The background clear color as `0xAARRGGBB`. Default opaque black. + public void setClearColor(int argb) { + synchronized (stateLock) { + this.clearColor = argb; + } + renderView.requestRender(); + } + + /// Controls whether the view renders continuously or only when + /// `#requestRender()` is called. Head tracked scenes normally want + /// continuous rendering. + public VRView setContinuous(boolean continuous) { + renderView.setContinuous(continuous); + return this; + } + + /// Requests that a single frame be rendered. + public void requestRender() { + renderView.requestRender(); + } + + /// The underlying render view, exposed for advanced integration. + public RenderView getRenderView() { + return renderView; + } + + @Override + protected void initComponent() { + super.initComponent(); + if (isHeadTrackingEnabled()) { + tracker.start(); + } + } + + @Override + protected void deinitialize() { + tracker.stop(); + super.deinitialize(); + } + + /// The internal `Renderer` that splits each frame into per-eye passes. + private final class EyeLoop implements Renderer { + @Override + public void onInit(GraphicsDevice device) { + vrRenderer.onInit(device); + } + + @Override + public void onResize(GraphicsDevice device, int width, int height) { + surfaceWidth = Math.max(1, width); + surfaceHeight = Math.max(1, height); + } + + @Override + public void onFrame(GraphicsDevice device) { + boolean trackingNow; + boolean stereoNow; + int clearNow; + float px; + float py; + float pz; + synchronized (stateLock) { + trackingNow = headTrackingEnabled; + stereoNow = stereo; + clearNow = clearColor; + px = posX; + py = posY; + pz = posZ; + } + if (trackingNow) { + tracker.getOrientation(quat); + rig.setOrientation(quat); + } + rig.setPosition(px, py, pz); + int w = surfaceWidth; + int h = surfaceHeight; + device.clear(clearNow, true, true); + if (stereoNow) { + int half = w / 2; + renderEye(device, VREye.LEFT, 0, 0, half, h); + renderEye(device, VREye.RIGHT, half, 0, w - half, h); + device.setViewport(0, 0, w, h); + } else { + renderEye(device, VREye.CENTER, 0, 0, w, h); + } + } + + private void renderEye(GraphicsDevice device, VREye eye, + int x, int y, int w, int h) { + device.setViewport(x, y, w, h); + rig.apply(eyeCamera, eye, (float) w / (float) Math.max(1, h)); + device.setCamera(eyeCamera); + vrRenderer.onEyeFrame(device, eye, eyeCamera); + } + + @Override + public void onDispose(GraphicsDevice device) { + vrRenderer.onDispose(device); + } + } +} diff --git a/CodenameOne/src/com/codename1/vr/package-info.java b/CodenameOne/src/com/codename1/vr/package-info.java new file mode 100644 index 00000000000..6cb513f5496 --- /dev/null +++ b/CodenameOne/src/com/codename1/vr/package-info.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/// Virtual reality and 360 media built entirely on the portable +/// `com.codename1.gpu` pipeline and `com.codename1.sensors` motion sensors - +/// no platform SDK dependency, so everything here runs wherever +/// `Display#isGpuSupported()` is true, including the simulator. +/// +/// `VRView` renders an application `VRRenderer` in side-by-side stereo with +/// head tracking; `Media360View` displays equirectangular panorama photos +/// with drag or gyroscope look-around. The building blocks are public too: +/// `OrientationFilter` (deterministic gyro/accel/magnetometer fusion), +/// `HeadTracker` (sensor wiring plus thread-safe snapshots) and +/// `VRCameraRig` (per-eye camera math). +/// +/// **Threading**: `VRRenderer` and `TextureSource` callbacks run on the +/// platform render thread; everything else is EDT-friendly component API. +package com.codename1.vr; diff --git a/Ports/Android/build.xml b/Ports/Android/build.xml index 6be43033831..3ef7fedd75f 100644 --- a/Ports/Android/build.xml +++ b/Ports/Android/build.xml @@ -110,10 +110,16 @@ + + classpath="${javac.classpath}:build/tmp" + excludes="com/codename1/impl/android/ar/**"> - + diff --git a/Ports/Android/nbproject/project.properties b/Ports/Android/nbproject/project.properties index 340e16a4f7d..ae84c3cc82f 100644 --- a/Ports/Android/nbproject/project.properties +++ b/Ports/Android/nbproject/project.properties @@ -26,7 +26,11 @@ dist.dir=dist dist.jar=${dist.dir}/Android.jar dist.javadoc.dir=${dist.dir}/javadoc endorsed.classpath= -excludes= +# The ARCore-backed AR impl compiles against com.google.ar.core which is not +# in cn1-binaries; it is compiled inside user app builds where the Android +# builder adds the ARCore gradle dependency (and deletes the package for +# non-AR apps). Mirrors the maven-compiler exclude in maven/android/pom.xml. +excludes=com/codename1/impl/android/ar/** file.reference.android-billing-4.0.0.jar=../../../cn1-binaries/android/android-billing-4.0.0.jar file.reference.android-support-v7-appcompat.jar=../../../cn1-binaries/android/android-support-v7-appcompat.jar file.reference.android-support-v7-cardview.jar=../../../cn1-binaries/android/android-support-v7-cardview.jar diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index c4694793ea9..ea0c3dc6301 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -10539,6 +10539,25 @@ public com.codename1.impl.CameraImpl createCameraImpl() { return new AndroidCameraImpl(act); } + @Override + public com.codename1.impl.ARImpl createARImpl() { + Activity act = getActivity(); + if (act == null) { + return null; + } + // The ARCore-backed impl lives in a package the build deletes for + // apps that never reference com.codename1.ar (it compiles against + // com.google.ar.core which only exists when the AR gradle dependency + // was injected), so it must be reached reflectively. + try { + Class clazz = Class.forName("com.codename1.impl.android.ar.AndroidARImpl"); + return (com.codename1.impl.ARImpl) clazz + .getConstructor(Activity.class).newInstance(act); + } catch (Throwable t) { + return null; + } + } + // Deeper-network connectivity platform factories. Each returns a small // platform-specific class living under // com.codename1.impl.android.connectivity. Those classes are loaded diff --git a/Ports/Android/src/com/codename1/impl/android/ar/ARCoreView.java b/Ports/Android/src/com/codename1/impl/android/ar/ARCoreView.java new file mode 100644 index 00000000000..58ccbabfe08 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ar/ARCoreView.java @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ar; + +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.opengl.GLUtils; +import android.opengl.Matrix; +import android.view.Display; +import android.view.WindowManager; + +import com.google.ar.core.Camera; +import com.google.ar.core.Coordinates2d; +import com.google.ar.core.Frame; +import com.google.ar.core.Session; +import com.google.ar.core.TrackingState; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +/** + * The AR view: a GLSurfaceView that renders the ARCore camera image as the + * background (via the external OES texture ARCore fills) and draws every + * anchored mesh on top with a minimal lit/textured GLES2 shader. ARCore has + * no built-in renderer, so this class owns the whole draw path and drives + * {@link AndroidARImpl#processFrame} once per frame. + */ +class ARCoreView extends GLSurfaceView implements GLSurfaceView.Renderer { + + private final AndroidARImpl impl; + private int cameraTextureId = -1; + private boolean displayGeometryDirty = true; + private int viewportWidth = 1; + private int viewportHeight = 1; + + // Camera background resources. + private int backgroundProgram; + private int backgroundPositionAttr; + private int backgroundTexCoordAttr; + private final FloatBuffer quadCoords; + private final FloatBuffer quadTexCoords; + + // Mesh shader resources. + private int meshProgram; + private int meshPositionAttr; + private int meshNormalAttr; + private int meshUvAttr; + private int meshMvpUniform; + private int meshModelUniform; + private int meshColorUniform; + private int meshUseTextureUniform; + private int meshTextureUniform; + private int meshLightDirUniform; + private int meshLightIntensityUniform; + + private volatile float lightIntensity = 1f; + + private static final String BACKGROUND_VERTEX = + "attribute vec4 a_Position;\n" + + "attribute vec2 a_TexCoord;\n" + + "varying vec2 v_TexCoord;\n" + + "void main() {\n" + + " gl_Position = a_Position;\n" + + " v_TexCoord = a_TexCoord;\n" + + "}\n"; + + private static final String BACKGROUND_FRAGMENT = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 v_TexCoord;\n" + + "uniform samplerExternalOES u_Texture;\n" + + "void main() {\n" + + " gl_FragColor = texture2D(u_Texture, v_TexCoord);\n" + + "}\n"; + + private static final String MESH_VERTEX = + "uniform mat4 u_Mvp;\n" + + "uniform mat4 u_Model;\n" + + "attribute vec4 a_Position;\n" + + "attribute vec3 a_Normal;\n" + + "attribute vec2 a_Uv;\n" + + "varying vec3 v_Normal;\n" + + "varying vec2 v_Uv;\n" + + "void main() {\n" + + " gl_Position = u_Mvp * a_Position;\n" + + " v_Normal = normalize((u_Model * vec4(a_Normal, 0.0)).xyz);\n" + + " v_Uv = a_Uv;\n" + + "}\n"; + + private static final String MESH_FRAGMENT = + "precision mediump float;\n" + + "uniform vec4 u_Color;\n" + + "uniform float u_UseTexture;\n" + + "uniform sampler2D u_Texture;\n" + + "uniform vec3 u_LightDir;\n" + + "uniform float u_LightIntensity;\n" + + "varying vec3 v_Normal;\n" + + "varying vec2 v_Uv;\n" + + "void main() {\n" + + " vec4 base = mix(u_Color, texture2D(u_Texture, v_Uv), u_UseTexture);\n" + + " float diffuse = max(dot(normalize(v_Normal), -u_LightDir), 0.0);\n" + + " float shade = clamp((0.4 + 0.6 * diffuse) * u_LightIntensity, 0.0, 1.5);\n" + + " gl_FragColor = vec4(base.rgb * shade, base.a);\n" + + "}\n"; + + ARCoreView(AndroidARImpl impl) { + super(impl.getActivity()); + this.impl = impl; + quadCoords = allocateFloats(new float[]{-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f}); + quadTexCoords = allocateFloats(new float[8]); + setPreserveEGLContextOnPause(true); + setEGLContextClientVersion(2); + setEGLConfigChooser(8, 8, 8, 8, 16, 0); + setRenderer(this); + setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); + setWillNotDraw(false); + } + + private static FloatBuffer allocateFloats(float[] values) { + FloatBuffer b = ByteBuffer.allocateDirect(values.length * 4) + .order(ByteOrder.nativeOrder()).asFloatBuffer(); + b.put(values); + b.rewind(); + return b; + } + + void pauseView() { + onPause(); + } + + void resumeView() { + onResume(); + } + + /** Marks a mesh entry's GL objects for recreation after its removal. */ + void recycleEntry(AndroidARImpl.MeshEntry entry) { + // GL objects leak until the context dies; acceptable for the + // placement-scale content this renderer targets. + entry.vbo = 0; + entry.ibo = 0; + entry.textureId = 0; + } + + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + GLES20.glClearColor(0f, 0f, 0f, 1f); + + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + cameraTextureId = textures[0]; + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + + backgroundProgram = buildProgram(BACKGROUND_VERTEX, BACKGROUND_FRAGMENT); + backgroundPositionAttr = GLES20.glGetAttribLocation(backgroundProgram, "a_Position"); + backgroundTexCoordAttr = GLES20.glGetAttribLocation(backgroundProgram, "a_TexCoord"); + + meshProgram = buildProgram(MESH_VERTEX, MESH_FRAGMENT); + meshPositionAttr = GLES20.glGetAttribLocation(meshProgram, "a_Position"); + meshNormalAttr = GLES20.glGetAttribLocation(meshProgram, "a_Normal"); + meshUvAttr = GLES20.glGetAttribLocation(meshProgram, "a_Uv"); + meshMvpUniform = GLES20.glGetUniformLocation(meshProgram, "u_Mvp"); + meshModelUniform = GLES20.glGetUniformLocation(meshProgram, "u_Model"); + meshColorUniform = GLES20.glGetUniformLocation(meshProgram, "u_Color"); + meshUseTextureUniform = GLES20.glGetUniformLocation(meshProgram, "u_UseTexture"); + meshTextureUniform = GLES20.glGetUniformLocation(meshProgram, "u_Texture"); + meshLightDirUniform = GLES20.glGetUniformLocation(meshProgram, "u_LightDir"); + meshLightIntensityUniform = GLES20.glGetUniformLocation(meshProgram, "u_LightIntensity"); + + Session session = impl.getSession(); + if (session != null) { + session.setCameraTextureName(cameraTextureId); + try { + session.resume(); + } catch (Throwable t) { + t.printStackTrace(); + } + } + } + + private static int buildProgram(String vertexSrc, String fragmentSrc) { + int vs = compileShader(GLES20.GL_VERTEX_SHADER, vertexSrc); + int fs = compileShader(GLES20.GL_FRAGMENT_SHADER, fragmentSrc); + int program = GLES20.glCreateProgram(); + GLES20.glAttachShader(program, vs); + GLES20.glAttachShader(program, fs); + GLES20.glLinkProgram(program); + return program; + } + + private static int compileShader(int type, String src) { + int shader = GLES20.glCreateShader(type); + GLES20.glShaderSource(shader, src); + GLES20.glCompileShader(shader); + return shader; + } + + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) { + viewportWidth = Math.max(1, width); + viewportHeight = Math.max(1, height); + GLES20.glViewport(0, 0, viewportWidth, viewportHeight); + displayGeometryDirty = true; + } + + @Override + public void onDrawFrame(GL10 gl) { + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); + Session session = impl.getSession(); + if (session == null || impl.isClosed()) { + return; + } + try { + if (displayGeometryDirty) { + displayGeometryDirty = false; + session.setDisplayGeometry(displayRotation(), viewportWidth, viewportHeight); + } + session.setCameraTextureName(cameraTextureId); + Frame frame = session.update(); + Camera camera = frame.getCamera(); + + if (frame.hasDisplayGeometryChanged()) { + frame.transformCoordinates2d( + Coordinates2d.OPENGL_NORMALIZED_DEVICE_COORDINATES, quadCoords, + Coordinates2d.TEXTURE_NORMALIZED, quadTexCoords); + } + drawCameraBackground(); + + impl.processFrame(frame, camera); + + if (camera.getTrackingState() == TrackingState.TRACKING) { + drawMeshes(camera); + } + } catch (Throwable t) { + t.printStackTrace(); + } + } + + private int displayRotation() { + try { + WindowManager wm = impl.getActivity().getWindowManager(); + Display d = wm.getDefaultDisplay(); + return d.getRotation(); + } catch (Throwable t) { + return 0; + } + } + + private void drawCameraBackground() { + GLES20.glDisable(GLES20.GL_DEPTH_TEST); + GLES20.glDepthMask(false); + GLES20.glUseProgram(backgroundProgram); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId); + quadCoords.rewind(); + GLES20.glVertexAttribPointer(backgroundPositionAttr, 2, GLES20.GL_FLOAT, + false, 0, quadCoords); + quadTexCoords.rewind(); + GLES20.glVertexAttribPointer(backgroundTexCoordAttr, 2, GLES20.GL_FLOAT, + false, 0, quadTexCoords); + GLES20.glEnableVertexAttribArray(backgroundPositionAttr); + GLES20.glEnableVertexAttribArray(backgroundTexCoordAttr); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glDisableVertexAttribArray(backgroundPositionAttr); + GLES20.glDisableVertexAttribArray(backgroundTexCoordAttr); + GLES20.glDepthMask(true); + GLES20.glEnable(GLES20.GL_DEPTH_TEST); + } + + private void drawMeshes(Camera camera) { + Map anchorPoses = new HashMap(); + List entries = impl.meshSnapshot(anchorPoses); + if (entries.isEmpty()) { + return; + } + float[] viewM = new float[16]; + float[] projM = new float[16]; + camera.getViewMatrix(viewM, 0); + camera.getProjectionMatrix(projM, 0, 0.05f, 100f); + float[] vp = new float[16]; + Matrix.multiplyMM(vp, 0, projM, 0, viewM, 0); + + GLES20.glUseProgram(meshProgram); + GLES20.glUniform3f(meshLightDirUniform, -0.35f, -0.85f, -0.35f); + GLES20.glUniform1f(meshLightIntensityUniform, lightIntensity); + + float[] model = new float[16]; + float[] mvp = new float[16]; + for (AndroidARImpl.MeshEntry entry : entries) { + float[] anchorPose = anchorPoses.get(entry.anchorId); + if (anchorPose == null) { + continue; + } + ensureUploaded(entry); + Matrix.multiplyMM(model, 0, anchorPose, 0, entry.anchorLocal16, 0); + Matrix.multiplyMM(mvp, 0, vp, 0, model, 0); + GLES20.glUniformMatrix4fv(meshMvpUniform, 1, false, mvp, 0); + GLES20.glUniformMatrix4fv(meshModelUniform, 1, false, model, 0); + float a = ((entry.argbColor >> 24) & 0xff) / 255f; + float r = ((entry.argbColor >> 16) & 0xff) / 255f; + float g = ((entry.argbColor >> 8) & 0xff) / 255f; + float b = (entry.argbColor & 0xff) / 255f; + GLES20.glUniform4f(meshColorUniform, r, g, b, a); + if (entry.textureId != 0) { + GLES20.glUniform1f(meshUseTextureUniform, 1f); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, entry.textureId); + GLES20.glUniform1i(meshTextureUniform, 0); + } else { + GLES20.glUniform1f(meshUseTextureUniform, 0f); + } + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, entry.vbo); + int stride = 8 * 4; + GLES20.glVertexAttribPointer(meshPositionAttr, 3, GLES20.GL_FLOAT, false, stride, 0); + GLES20.glVertexAttribPointer(meshNormalAttr, 3, GLES20.GL_FLOAT, false, stride, 12); + GLES20.glVertexAttribPointer(meshUvAttr, 2, GLES20.GL_FLOAT, false, stride, 24); + GLES20.glEnableVertexAttribArray(meshPositionAttr); + GLES20.glEnableVertexAttribArray(meshNormalAttr); + GLES20.glEnableVertexAttribArray(meshUvAttr); + GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, entry.ibo); + GLES20.glDrawElements(GLES20.GL_TRIANGLES, entry.indices.length, + GLES20.GL_UNSIGNED_INT, 0); + GLES20.glDisableVertexAttribArray(meshPositionAttr); + GLES20.glDisableVertexAttribArray(meshNormalAttr); + GLES20.glDisableVertexAttribArray(meshUvAttr); + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0); + GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, 0); + } + } + + /** Feeds the light estimate through so drawn content matches the scene. */ + void setLightIntensity(float intensity) { + this.lightIntensity = intensity; + } + + private void ensureUploaded(AndroidARImpl.MeshEntry entry) { + if (entry.vbo == 0) { + int[] ids = new int[2]; + GLES20.glGenBuffers(2, ids, 0); + entry.vbo = ids[0]; + entry.ibo = ids[1]; + int floatCount = entry.vertexCount * 8; + FloatBuffer vb = ByteBuffer.allocateDirect(floatCount * 4) + .order(ByteOrder.nativeOrder()).asFloatBuffer(); + vb.put(entry.interleaved, 0, floatCount); + vb.rewind(); + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, entry.vbo); + GLES20.glBufferData(GLES20.GL_ARRAY_BUFFER, floatCount * 4, vb, + GLES20.GL_STATIC_DRAW); + IntBuffer ib = ByteBuffer.allocateDirect(entry.indices.length * 4) + .order(ByteOrder.nativeOrder()).asIntBuffer(); + ib.put(entry.indices); + ib.rewind(); + GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, entry.ibo); + GLES20.glBufferData(GLES20.GL_ELEMENT_ARRAY_BUFFER, entry.indices.length * 4, ib, + GLES20.GL_STATIC_DRAW); + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0); + GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, 0); + if (entry.texture != null && entry.textureId == 0) { + int[] tex = new int[1]; + GLES20.glGenTextures(1, tex, 0); + entry.textureId = tex[0]; + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, entry.textureId); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, + GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, + GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, entry.texture, 0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + } + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ar/AndroidARImpl.java b/Ports/Android/src/com/codename1/impl/android/ar/AndroidARImpl.java new file mode 100644 index 00000000000..f53cb1e6d76 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ar/AndroidARImpl.java @@ -0,0 +1,859 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ar; + +import android.Manifest; +import android.app.Activity; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; + +import com.codename1.ar.ARCapabilities; +import com.codename1.ar.ARFaceAnchor; +import com.codename1.ar.ARFaceRegion; +import com.codename1.ar.ARHitResult; +import com.codename1.ar.ARImageAnchor; +import com.codename1.ar.ARLightEstimate; +import com.codename1.ar.ARModel; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARPose; +import com.codename1.ar.ARReferenceImage; +import com.codename1.ar.ARSessionOptions; +import com.codename1.ar.ARTrackingFailureReason; +import com.codename1.ar.ARTrackingMode; +import com.codename1.ar.ARTrackingState; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Quaternion; +import com.codename1.impl.ARImpl; +import com.codename1.impl.android.AndroidImplementation; +import com.codename1.ui.Display; +import com.codename1.ui.EncodedImage; +import com.codename1.ui.Image; +import com.codename1.ui.PeerComponent; +import com.codename1.util.AsyncResource; + +import com.google.ar.core.Anchor; +import com.google.ar.core.ArCoreApk; +import com.google.ar.core.AugmentedFace; +import com.google.ar.core.AugmentedImage; +import com.google.ar.core.AugmentedImageDatabase; +import com.google.ar.core.Camera; +import com.google.ar.core.Config; +import com.google.ar.core.Frame; +import com.google.ar.core.HitResult; +import com.google.ar.core.LightEstimate; +import com.google.ar.core.Plane; +import com.google.ar.core.Pose; +import com.google.ar.core.Session; +import com.google.ar.core.TrackingFailureReason; +import com.google.ar.core.TrackingState; + +import java.io.IOException; +import java.nio.FloatBuffer; +import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +/** + * Android implementation of {@link ARImpl} backed by ARCore (Google Play + * Services for AR). This package compiles against {@code com.google.ar.core} + * which only exists when the build detects {@code com.codename1.ar} usage and + * adds the ARCore Gradle dependency; for non-AR apps the whole + * {@code com.codename1.impl.android.ar} package is deleted from the generated + * project, so {@code AndroidImplementation} reaches this class through + * reflection only. + * + * ARCore supplies tracking but no renderer: {@link ARCoreView} owns the + * GLSurfaceView that draws the camera background and the anchored content, + * and drives {@link #processFrame(Frame, Camera)} once per rendered frame. + */ +public class AndroidARImpl extends ARImpl { + + private final Activity activity; + private EventSink sink; + private Session session; + private ARSessionOptions options; + private ARCoreView view; + private volatile boolean closed; + + private final Object sceneLock = new Object(); + private final Map anchors = new HashMap(); + private final Map planeIds = new HashMap(); + private final Map planeSnapshots = new HashMap(); + private final Map imageIds = new HashMap(); + private final Map faceIds = new HashMap(); + private final Map imagesById = new HashMap(); + private final Map facesById = new HashMap(); + private final List pendingHits = new ArrayList(); + private final List recentHits = new ArrayList(); + private final List meshes = new ArrayList(); + private final Map textureCache = new HashMap(); + private int idSeq; + private int frameCounter; + private int lastTrackingCode = -1; + private int lastReasonCode = -1; + + private static final class PendingHit { + final float xNorm; + final float yNorm; + final AsyncResource result; + + PendingHit(float xNorm, float yNorm, AsyncResource result) { + this.xNorm = xNorm; + this.yNorm = yNorm; + this.result = result; + } + } + + /** + * One flattened renderable: an anchored mesh with its transform relative + * to the anchor. {@link ARCoreView} uploads the GL buffers lazily on the + * render thread and stores the handles here. + */ + static final class MeshEntry { + final String anchorId; + final float[] interleaved; + final int vertexCount; + final int[] indices; + final int argbColor; + final Bitmap texture; + final float[] anchorLocal16; + // GL handles owned by the render thread. + int vbo; + int ibo; + int textureId; + + MeshEntry(String anchorId, float[] interleaved, int vertexCount, int[] indices, + int argbColor, Bitmap texture, float[] anchorLocal16) { + this.anchorId = anchorId; + this.interleaved = interleaved; + this.vertexCount = vertexCount; + this.indices = indices; + this.argbColor = argbColor; + this.texture = texture; + this.anchorLocal16 = anchorLocal16; + } + } + + public AndroidARImpl(Activity activity) { + this.activity = activity; + } + + @Override + public ARCapabilities getCapabilities() { + boolean supported; + try { + ArCoreApk.Availability availability = + ArCoreApk.getInstance().checkAvailability(activity); + supported = availability.isSupported() + || availability == ArCoreApk.Availability.UNKNOWN_CHECKING; + } catch (Throwable t) { + supported = false; + } + return new ARCapabilities(supported, supported, supported, supported, supported); + } + + @Override + public void setEventSink(EventSink sink) { + this.sink = sink; + } + + @Override + public void open(ARSessionOptions opts) throws IOException { + this.options = opts; + if (!AndroidImplementation.checkForPermission(Manifest.permission.CAMERA, + "AR needs the camera")) { + throw new IOException("The camera permission was denied"); + } + try { + ArCoreApk.InstallStatus install = + ArCoreApk.getInstance().requestInstall(activity, true); + if (install == ArCoreApk.InstallStatus.INSTALL_REQUESTED) { + // Play Services for AR installation was launched; the app + // resumes afterwards and should retry AR.open then. + throw new IOException( + "ARCore installation was requested; retry once it completes"); + } + if (opts.getTrackingMode() == ARTrackingMode.FACE) { + session = new Session(activity, EnumSet.of(Session.Feature.FRONT_CAMERA)); + } else { + session = new Session(activity); + } + Config config = new Config(session); + config.setUpdateMode(Config.UpdateMode.LATEST_CAMERA_IMAGE); + config.setFocusMode(Config.FocusMode.AUTO); + config.setLightEstimationMode(opts.isLightEstimation() + ? Config.LightEstimationMode.AMBIENT_INTENSITY + : Config.LightEstimationMode.DISABLED); + if (opts.getTrackingMode() == ARTrackingMode.FACE) { + config.setAugmentedFaceMode(Config.AugmentedFaceMode.MESH3D); + config.setPlaneFindingMode(Config.PlaneFindingMode.DISABLED); + } else { + boolean h = opts.getPlaneDetection().includesHorizontal(); + boolean v = opts.getPlaneDetection().includesVertical(); + config.setPlaneFindingMode(h && v + ? Config.PlaneFindingMode.HORIZONTAL_AND_VERTICAL + : v ? Config.PlaneFindingMode.VERTICAL + : h ? Config.PlaneFindingMode.HORIZONTAL + : Config.PlaneFindingMode.DISABLED); + ARReferenceImage[] images = opts.getReferenceImages(); + if (images.length > 0) { + AugmentedImageDatabase db = new AugmentedImageDatabase(session); + for (ARReferenceImage img : images) { + byte[] bytes = img.getEncodedImage(); + Bitmap bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.length); + if (bitmap != null) { + db.addImage(img.getName(), bitmap, img.getPhysicalWidthMeters()); + } + } + config.setAugmentedImageDatabase(db); + } + } + session.configure(config); + } catch (IOException e) { + throw e; + } catch (Throwable t) { + closeSessionQuietly(); + throw new IOException("Could not start ARCore: " + t, t); + } + } + + private void closeSessionQuietly() { + Session s = session; + session = null; + if (s != null) { + try { + s.close(); + } catch (Throwable ignore) { + } + } + } + + Session getSession() { + return session; + } + + Activity getActivity() { + return activity; + } + + @Override + public PeerComponent createViewPeer() { + if (session == null) { + return null; + } + // The GLSurfaceView must be constructed on the Android UI thread. + final ARCoreView[] holder = new ARCoreView[1]; + final CountDownLatch latch = new CountDownLatch(1); + activity.runOnUiThread(new Runnable() { + public void run() { + try { + holder[0] = new ARCoreView(AndroidARImpl.this); + } catch (Throwable t) { + t.printStackTrace(); + } finally { + latch.countDown(); + } + } + }); + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + view = holder[0]; + if (view == null) { + return null; + } + return PeerComponent.create(view); + } + + @Override + public void hitTest(float xNorm, float yNorm, AsyncResource result) { + synchronized (sceneLock) { + if (closed || session == null) { + completeHits(result, new ARHitResult[0]); + return; + } + // ARCore hit tests need a Frame; queue for the render loop. + pendingHits.add(new PendingHit(xNorm, yNorm, result)); + } + } + + private void completeHits(AsyncResource result, ARHitResult[] hits) { + Display.getInstance().callSerially(new CompleteOnEdt(result, hits)); + } + + /** + * Resolves an AsyncResource on the EDT. A named static class since it + * only needs the resource and value, not the enclosing impl. + */ + private static final class CompleteOnEdt implements Runnable { + private final AsyncResource result; + private final T value; + + CompleteOnEdt(AsyncResource result, T value) { + this.result = result; + this.value = value; + } + + public void run() { + result.complete(value); + } + } + + @Override + public String createAnchor(ARPose pose) { + return registerAnchor(session.createAnchor(toPose(pose))); + } + + @Override + public String createAnchorFromHit(Object nativeHandle, ARPose pose) { + if (nativeHandle instanceof HitResult) { + synchronized (sceneLock) { + if (recentHits.contains(nativeHandle)) { + try { + return registerAnchor(((HitResult) nativeHandle).createAnchor()); + } catch (Throwable t) { + // Fall back to a plain pose anchor below. + } + } + } + } + return createAnchor(pose); + } + + private String registerAnchor(Anchor anchor) { + String id = "arcore-anchor-" + (++idSeq); + synchronized (sceneLock) { + anchors.put(id, anchor); + } + return id; + } + + private static Pose toPose(ARPose pose) { + return new Pose( + new float[]{pose.getTx(), pose.getTy(), pose.getTz()}, + new float[]{pose.getQx(), pose.getQy(), pose.getQz(), pose.getQw()}); + } + + private static ARPose fromPose(Pose pose) { + float[] t = new float[3]; + float[] q = new float[4]; + pose.getTranslation(t, 0); + pose.getRotationQuaternion(q, 0); + return new ARPose(t[0], t[1], t[2], q[0], q[1], q[2], q[3]); + } + + @Override + public void removeAnchor(String anchorId) { + Anchor anchor; + synchronized (sceneLock) { + anchor = anchors.remove(anchorId); + removeMeshesFor(anchorId); + } + if (anchor != null) { + anchor.detach(); + } + } + + private void removeMeshesFor(String anchorId) { + for (int i = meshes.size() - 1; i >= 0; i--) { + if (meshes.get(i).anchorId.equals(anchorId)) { + MeshEntry dead = meshes.remove(i); + if (view != null) { + view.recycleEntry(dead); + } + } + } + } + + @Override + public void setAnchorNode(String anchorId, ARNode node) { + synchronized (sceneLock) { + removeMeshesFor(anchorId); + if (node != null) { + flatten(anchorId, node, Matrix4.identity()); + } + } + } + + @Override + public void nodeChanged(String anchorId, ARNode node) { + setAnchorNode(anchorId, node); + } + + private void flatten(String anchorId, ARNode node, float[] parentMatrix) { + if (!node.isVisible()) { + return; + } + float[] local = Matrix4.translation(node.getLocalX(), node.getLocalY(), node.getLocalZ()); + float[] rot = new float[16]; + Quaternion.toMatrix(new float[]{node.getLocalQx(), node.getLocalQy(), + node.getLocalQz(), node.getLocalQw()}, rot); + float[] tmp = new float[16]; + Matrix4.multiply(local, rot, tmp); + float s = node.getLocalScale(); + float[] m = new float[16]; + Matrix4.multiply(tmp, Matrix4.scaling(s, s, s), m); + float[] world = new float[16]; + Matrix4.multiply(parentMatrix, m, world); + + ARModel model = node.getModel(); + if (model != null) { + Mesh mesh = model.getMesh(); + if (mesh != null && mesh.getVertices().getFormat().getFloatsPerVertex() == 8) { + int vertexCount = mesh.getVertices().getVertexCount(); + int[] indices; + if (mesh.getIndices() != null) { + short[] shortIdx = mesh.getIndices().getData(); + indices = new int[mesh.getIndices().getIndexCount()]; + for (int i = 0; i < indices.length; i++) { + indices[i] = shortIdx[i] & 0xffff; + } + } else { + indices = new int[vertexCount]; + for (int i = 0; i < vertexCount; i++) { + indices[i] = i; + } + } + meshes.add(new MeshEntry(anchorId, mesh.getVertices().getData(), vertexCount, + indices, model.getColor(), textureBitmap(model), world)); + } + } + for (int i = 0; i < node.getChildCount(); i++) { + flatten(anchorId, node.getChildAt(i), world); + } + } + + private Bitmap textureBitmap(ARModel model) { + if (textureCache.containsKey(model)) { + return textureCache.get(model); + } + Bitmap bitmap = null; + Image img = model.getBaseColorImage(); + if (img != null) { + byte[] bytes; + if (img instanceof EncodedImage) { + bytes = ((EncodedImage) img).getImageData(); + } else { + bytes = EncodedImage.createFromImage(img, false).getImageData(); + } + bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.length); + } + textureCache.put(model, bitmap); + return bitmap; + } + + @Override + public void requestPermissions(AsyncResource result) { + boolean granted = AndroidImplementation.checkForPermission( + Manifest.permission.CAMERA, "AR needs the camera"); + Display.getInstance().callSerially( + new CompleteOnEdt(result, Boolean.valueOf(granted))); + } + + @Override + public void pause() { + if (view != null) { + view.pauseView(); + } + Session s = session; + if (s != null) { + s.pause(); + } + } + + @Override + public void resume() { + Session s = session; + if (s != null) { + try { + s.resume(); + } catch (Throwable t) { + t.printStackTrace(); + } + } + if (view != null) { + view.resumeView(); + } + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + List orphans; + synchronized (sceneLock) { + orphans = new ArrayList(pendingHits); + pendingHits.clear(); + anchors.clear(); + meshes.clear(); + recentHits.clear(); + } + for (PendingHit hit : orphans) { + completeHits(hit.result, new ARHitResult[0]); + } + if (view != null) { + view.pauseView(); + } + Session s = session; + session = null; + if (s != null) { + try { + s.pause(); + } catch (Throwable ignore) { + } + try { + s.close(); + } catch (Throwable ignore) { + } + } + } + + boolean isClosed() { + return closed; + } + + /** + * Snapshot of the anchored meshes plus their current anchor poses, taken + * by the render thread each frame. + */ + List meshSnapshot(Map anchorPoseMatrices) { + synchronized (sceneLock) { + for (Map.Entry e : anchors.entrySet()) { + if (e.getValue().getTrackingState() == TrackingState.TRACKING) { + float[] m = new float[16]; + e.getValue().getPose().toMatrix(m, 0); + anchorPoseMatrices.put(e.getKey(), m); + } + } + for (Map.Entry e : imagesById.entrySet()) { + float[] m = new float[16]; + e.getValue().getCenterPose().toMatrix(m, 0); + anchorPoseMatrices.put(e.getKey(), m); + } + for (Map.Entry e : facesById.entrySet()) { + float[] m = new float[16]; + e.getValue().getCenterPose().toMatrix(m, 0); + anchorPoseMatrices.put(e.getKey(), m); + } + return new ArrayList(meshes); + } + } + + /** + * Called by {@link ARCoreView} on the GL thread after every + * {@code session.update()}: publishes tracking / plane / anchor / image / + * face / light events through the sink (which marshals to the EDT) and + * services queued hit tests. + */ + void processFrame(Frame frame, Camera camera) { + if (closed || sink == null) { + return; + } + publishTracking(camera); + servicePendingHits(frame, camera); + if (options.getTrackingMode() == ARTrackingMode.FACE) { + processFaces(frame); + } else { + processPlanes(frame); + processImages(frame); + } + publishAnchors(); + frameCounter++; + if (frameCounter % 6 == 0) { + sink.onCameraPose(fromPose(camera.getDisplayOrientedPose())); + publishLight(frame); + } + } + + private void publishTracking(Camera camera) { + int state; + int reason = 0; + TrackingState ts = camera.getTrackingState(); + if (ts == TrackingState.TRACKING) { + state = 2; + } else if (ts == TrackingState.PAUSED) { + state = 1; + TrackingFailureReason r = camera.getTrackingFailureReason(); + if (r == TrackingFailureReason.EXCESSIVE_MOTION) { + reason = 2; + } else if (r == TrackingFailureReason.INSUFFICIENT_LIGHT) { + reason = 3; + } else if (r == TrackingFailureReason.INSUFFICIENT_FEATURES) { + reason = 4; + } else { + reason = 1; + } + } else { + state = 0; + } + if (state != lastTrackingCode || reason != lastReasonCode) { + lastTrackingCode = state; + lastReasonCode = reason; + ARTrackingState s = state == 2 ? ARTrackingState.TRACKING + : state == 1 ? ARTrackingState.LIMITED : ARTrackingState.NOT_TRACKING; + ARTrackingFailureReason fr = reason == 2 ? ARTrackingFailureReason.EXCESSIVE_MOTION + : reason == 3 ? ARTrackingFailureReason.INSUFFICIENT_LIGHT + : reason == 4 ? ARTrackingFailureReason.INSUFFICIENT_FEATURES + : reason == 1 ? ARTrackingFailureReason.INITIALIZING + : ARTrackingFailureReason.NONE; + sink.onTrackingStateChanged(s, fr); + } + } + + private void servicePendingHits(Frame frame, Camera camera) { + List hits = null; + synchronized (sceneLock) { + if (!pendingHits.isEmpty()) { + hits = new ArrayList(pendingHits); + pendingHits.clear(); + } + } + if (hits == null) { + return; + } + int w = view == null ? 1 : Math.max(1, view.getWidth()); + int h = view == null ? 1 : Math.max(1, view.getHeight()); + for (PendingHit pending : hits) { + List out = new ArrayList(); + if (camera.getTrackingState() == TrackingState.TRACKING) { + try { + List results = frame.hitTest(pending.xNorm * w, pending.yNorm * h); + for (HitResult hit : results) { + if (hit.getTrackable() instanceof Plane) { + Plane plane = (Plane) hit.getTrackable(); + if (!plane.isPoseInPolygon(hit.getHitPose())) { + continue; + } + synchronized (sceneLock) { + recentHits.add(hit); + } + String planeId = planeIds.get(plane); + out.add(new ARHitResult(fromPose(hit.getHitPose()), + hit.getDistance(), ARHitResult.Type.PLANE, + planeId == null ? null : planeSnapshots.get(planeId), + hit)); + } else { + synchronized (sceneLock) { + recentHits.add(hit); + } + out.add(new ARHitResult(fromPose(hit.getHitPose()), + hit.getDistance(), ARHitResult.Type.FEATURE_POINT, + null, hit)); + } + } + } catch (Throwable t) { + // Deliver what was collected; a torn-down session mid-test + // simply yields no hits. + } + } + // Cap the retained native hit handles. + synchronized (sceneLock) { + while (recentHits.size() > 32) { + recentHits.remove(0); + } + } + completeHits(pending.result, out.toArray(new ARHitResult[out.size()])); + } + } + + private void processPlanes(Frame frame) { + for (Plane plane : frame.getUpdatedTrackables(Plane.class)) { + if (plane.getSubsumedBy() != null) { + String id = planeIds.remove(plane); + if (id != null) { + planeSnapshots.remove(id); + sink.onPlaneRemoved(id); + } + continue; + } + if (plane.getTrackingState() == TrackingState.STOPPED) { + String id = planeIds.remove(plane); + if (id != null) { + planeSnapshots.remove(id); + sink.onPlaneRemoved(id); + } + continue; + } + boolean added = false; + String id = planeIds.get(plane); + if (id == null) { + id = "arcore-plane-" + (++idSeq); + planeIds.put(plane, id); + added = true; + } + ARPlane.Type type = plane.getType() == Plane.Type.VERTICAL + ? ARPlane.Type.VERTICAL + : plane.getType() == Plane.Type.HORIZONTAL_DOWNWARD_FACING + ? ARPlane.Type.HORIZONTAL_DOWN : ARPlane.Type.HORIZONTAL_UP; + float[] polygon = null; + FloatBuffer poly = plane.getPolygon(); + if (poly != null) { + polygon = new float[poly.limit()]; + poly.rewind(); + poly.get(polygon); + } + ARPlane snapshot = new ARPlane(id, type, fromPose(plane.getCenterPose()), + plane.getExtentX(), plane.getExtentZ(), polygon, + plane.getTrackingState() == TrackingState.TRACKING + ? ARTrackingState.TRACKING : ARTrackingState.LIMITED); + planeSnapshots.put(id, snapshot); + if (added) { + sink.onPlaneAdded(snapshot); + } else { + sink.onPlaneUpdated(snapshot); + } + } + } + + private void processImages(Frame frame) { + for (AugmentedImage image : frame.getUpdatedTrackables(AugmentedImage.class)) { + String id = imageIds.get(image); + if (image.getTrackingState() == TrackingState.STOPPED) { + if (id != null) { + imageIds.remove(image); + synchronized (sceneLock) { + imagesById.remove(id); + } + sink.onAnchorRemoved(id); + } + continue; + } + if (image.getTrackingState() != TrackingState.TRACKING) { + continue; + } + if (id == null) { + id = "arcore-image-" + (++idSeq); + imageIds.put(image, id); + synchronized (sceneLock) { + imagesById.put(id, image); + } + sink.onAnchorAdded(new ARImageAnchor(id, fromPose(image.getCenterPose()), + image.getName(), image.getExtentX())); + } else { + sink.onAnchorUpdated(id, fromPose(image.getCenterPose()), + ARTrackingState.TRACKING); + } + } + } + + private void processFaces(Frame frame) { + for (AugmentedFace face : frame.getUpdatedTrackables(AugmentedFace.class)) { + String id = faceIds.get(face); + if (face.getTrackingState() == TrackingState.STOPPED) { + if (id != null) { + faceIds.remove(face); + synchronized (sceneLock) { + facesById.remove(id); + } + sink.onAnchorRemoved(id); + } + continue; + } + if (face.getTrackingState() != TrackingState.TRACKING) { + continue; + } + boolean added = false; + if (id == null) { + id = "arcore-face-" + (++idSeq); + faceIds.put(face, id); + synchronized (sceneLock) { + facesById.put(id, face); + } + added = true; + sink.onAnchorAdded(new ARFaceAnchor(id, fromPose(face.getCenterPose()))); + } + // Region poses: ARCore natively supplies nose + forehead regions. + ARPose[] regions = new ARPose[ARFaceRegion.values().length]; + regions[ARFaceRegion.NOSE_TIP.ordinal()] = + fromPose(face.getRegionPose(AugmentedFace.RegionType.NOSE_TIP)); + regions[ARFaceRegion.FOREHEAD_LEFT.ordinal()] = + fromPose(face.getRegionPose(AugmentedFace.RegionType.FOREHEAD_LEFT)); + regions[ARFaceRegion.FOREHEAD_RIGHT.ordinal()] = + fromPose(face.getRegionPose(AugmentedFace.RegionType.FOREHEAD_RIGHT)); + float[] meshVertices = null; + int[] meshTriangles = null; + if (added || frameCounter % 6 == 0) { + FloatBuffer v = face.getMeshVertices(); + if (v != null) { + meshVertices = new float[v.limit()]; + v.rewind(); + v.get(meshVertices); + } + ShortBuffer tri = face.getMeshTriangleIndices(); + if (tri != null) { + meshTriangles = new int[tri.limit()]; + tri.rewind(); + for (int i = 0; i < meshTriangles.length; i++) { + meshTriangles[i] = tri.get(i); + } + } + } + sink.onFaceAnchorUpdated(id, fromPose(face.getCenterPose()), + ARTrackingState.TRACKING, regions, meshVertices, meshTriangles); + } + } + + private void publishAnchors() { + if (frameCounter % 6 != 0) { + return; + } + List updates = new ArrayList(); + synchronized (sceneLock) { + for (Map.Entry e : anchors.entrySet()) { + Anchor anchor = e.getValue(); + ARTrackingState state = anchor.getTrackingState() == TrackingState.TRACKING + ? ARTrackingState.TRACKING + : anchor.getTrackingState() == TrackingState.PAUSED + ? ARTrackingState.LIMITED : ARTrackingState.NOT_TRACKING; + updates.add(new Object[]{e.getKey(), fromPose(anchor.getPose()), state}); + } + } + for (Object[] u : updates) { + sink.onAnchorUpdated((String) u[0], (ARPose) u[1], (ARTrackingState) u[2]); + } + } + + private void publishLight(Frame frame) { + LightEstimate estimate = frame.getLightEstimate(); + if (estimate != null && estimate.getState() == LightEstimate.State.VALID) { + // ARCore's pixel intensity averages about 0.18 in neutral indoor + // lighting; normalize so 1.0 is neutral (the ARLightEstimate + // convention). + float intensity = estimate.getPixelIntensity() / 0.18f; + sink.onLightEstimate(new ARLightEstimate(true, intensity, 1f, 1f, 1f)); + if (view != null) { + view.setLightIntensity(intensity); + } + } + } +} diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/ARSimulation.java b/Ports/JavaSE/src/com/codename1/impl/javase/ARSimulation.java new file mode 100644 index 00000000000..c424c122d4f --- /dev/null +++ b/Ports/JavaSE/src/com/codename1/impl/javase/ARSimulation.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codename1.impl.javase; + +import com.codename1.ar.ARTrackingFailureReason; +import com.codename1.ar.ARTrackingMode; +import com.codename1.ar.ARTrackingState; + +import javax.swing.BorderFactory; +import javax.swing.JButton; +import javax.swing.JComboBox; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JOptionPane; +import javax.swing.JPanel; +import javax.swing.JSlider; +import java.awt.BorderLayout; +import java.awt.GridLayout; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; + +/** + * Simulator window that drives {@link JavaSEARImpl}, the simulated AR + * backend. Lets developers exercise the failure and recognition paths that + * are hard to hit on demand with real hardware: degraded tracking states, + * changing light estimates, reference image detection and face appearance. + * The virtual camera itself is driven directly on the AR view (mouse drag to + * look, WASD / arrows to move, Q / E for up / down). + * + * @author Codename One + */ +public class ARSimulation extends JFrame { + private final JComboBox trackingState = + new JComboBox(ARTrackingState.values()); + private final JComboBox failureReason = + new JComboBox(ARTrackingFailureReason.values()); + private final JSlider intensity = new JSlider(0, 200, 100); + private final JSlider red = new JSlider(50, 150, 100); + private final JSlider green = new JSlider(50, 150, 100); + private final JSlider blue = new JSlider(50, 150, 100); + + public ARSimulation() { + super("AR Simulation"); + setDefaultCloseOperation(JFrame.HIDE_ON_CLOSE); + setLayout(new BorderLayout()); + + JPanel top = new JPanel(new GridLayout(3, 1)); + JLabel help = new JLabel("Virtual camera: drag the AR view to look around," + + "
WASD / arrow keys move, Q / E go up / down." + + "
Controls below act on the currently open AR session."); + help.setBorder(BorderFactory.createEmptyBorder(8, 8, 8, 8)); + top.add(help); + + JPanel tracking = new JPanel(new GridLayout(1, 3, 6, 6)); + tracking.setBorder(BorderFactory.createTitledBorder("Tracking state")); + tracking.add(trackingState); + tracking.add(failureReason); + trackingState.setSelectedItem(ARTrackingState.TRACKING); + failureReason.setSelectedItem(ARTrackingFailureReason.NONE); + JButton applyTracking = new JButton("Apply"); + applyTracking.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + JavaSEARImpl impl = requireSession(); + if (impl != null) { + impl.simSetTrackingState( + (ARTrackingState) trackingState.getSelectedItem(), + (ARTrackingFailureReason) failureReason.getSelectedItem()); + } + } + }); + tracking.add(applyTracking); + top.add(tracking); + + JPanel actions = new JPanel(new GridLayout(1, 3, 6, 6)); + actions.setBorder(BorderFactory.createTitledBorder("Scene")); + JButton redetect = new JButton("Re-detect planes"); + redetect.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + JavaSEARImpl impl = requireSession(); + if (impl != null) { + impl.simRedetectPlanes(); + } + } + }); + actions.add(redetect); + JButton detectImage = new JButton("Detect reference image..."); + detectImage.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + JavaSEARImpl impl = requireSession(); + if (impl == null) { + return; + } + String[] names = impl.simGetReferenceImageNames(); + if (names.length == 0) { + JOptionPane.showMessageDialog(ARSimulation.this, + "The open session registered no reference images.\n" + + "Pass them via ARSessionOptions.referenceImages(...)."); + return; + } + String choice = (String) JOptionPane.showInputDialog(ARSimulation.this, + "Simulate the camera recognizing:", "Detect reference image", + JOptionPane.PLAIN_MESSAGE, null, names, names[0]); + if (choice != null) { + impl.simDetectImage(choice); + } + } + }); + actions.add(detectImage); + JButton toggleFace = new JButton("Toggle simulated face"); + toggleFace.addActionListener(new ActionListener() { + public void actionPerformed(ActionEvent e) { + JavaSEARImpl impl = requireSession(); + if (impl == null) { + return; + } + if (impl.getOpenOptions().getTrackingMode() != ARTrackingMode.FACE) { + JOptionPane.showMessageDialog(ARSimulation.this, + "The open session is not in FACE tracking mode.\n" + + "Open it with ARSessionOptions.trackingMode(ARTrackingMode.FACE)."); + return; + } + impl.simSetFaceVisible(!impl.simIsFaceVisible()); + } + }); + actions.add(toggleFace); + + JPanel lightPanel = new JPanel(new GridLayout(4, 1)); + lightPanel.setBorder(BorderFactory.createTitledBorder( + "Light estimate (100 = neutral)")); + lightPanel.add(labeledSlider("Intensity", intensity)); + lightPanel.add(labeledSlider("Red", red)); + lightPanel.add(labeledSlider("Green", green)); + lightPanel.add(labeledSlider("Blue", blue)); + javax.swing.event.ChangeListener lightListener = new javax.swing.event.ChangeListener() { + public void stateChanged(javax.swing.event.ChangeEvent e) { + JavaSEARImpl impl = JavaSEARImpl.activeInstance; + if (impl != null) { + impl.simSetLight(intensity.getValue() / 100f, red.getValue() / 100f, + green.getValue() / 100f, blue.getValue() / 100f); + } + } + }; + intensity.addChangeListener(lightListener); + red.addChangeListener(lightListener); + green.addChangeListener(lightListener); + blue.addChangeListener(lightListener); + + add(BorderLayout.NORTH, top); + add(BorderLayout.CENTER, actions); + add(BorderLayout.SOUTH, lightPanel); + pack(); + setLocationByPlatform(true); + } + + private JPanel labeledSlider(String label, JSlider slider) { + JPanel p = new JPanel(new BorderLayout(6, 0)); + p.add(BorderLayout.WEST, new JLabel(label)); + p.add(BorderLayout.CENTER, slider); + return p; + } + + private JavaSEARImpl requireSession() { + JavaSEARImpl impl = JavaSEARImpl.activeInstance; + if (impl == null) { + JOptionPane.showMessageDialog(this, + "No AR session is open. Open one in the app with AR.open(...)."); + } + return impl; + } +} diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEARImpl.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEARImpl.java new file mode 100644 index 00000000000..6d2ce08b991 --- /dev/null +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEARImpl.java @@ -0,0 +1,896 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codename1.impl.javase; + +import com.codename1.ar.ARAnchor; +import com.codename1.ar.ARCapabilities; +import com.codename1.ar.ARFaceAnchor; +import com.codename1.ar.ARFaceRegion; +import com.codename1.ar.ARHitResult; +import com.codename1.ar.ARImageAnchor; +import com.codename1.ar.ARLightEstimate; +import com.codename1.ar.ARModel; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARPose; +import com.codename1.ar.ARReferenceImage; +import com.codename1.ar.ARSessionOptions; +import com.codename1.ar.ARTrackingFailureReason; +import com.codename1.ar.ARTrackingMode; +import com.codename1.ar.ARTrackingState; +import com.codename1.gpu.Camera; +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Light; +import com.codename1.gpu.Material; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Primitives; +import com.codename1.gpu.Quaternion; +import com.codename1.gpu.Texture; +import com.codename1.impl.ARImpl; +import com.codename1.ui.Display; +import com.codename1.ui.PeerComponent; +import com.codename1.util.AsyncResource; + +import javax.swing.JComponent; +import java.awt.Graphics; +import java.awt.event.KeyAdapter; +import java.awt.event.KeyEvent; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.awt.image.BufferedImage; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.TimerTask; + +/// Simulated AR backend for the Codename One simulator, modeled on the +/// Android emulator's ARCore "virtual scene": the session runs against a +/// virtual room whose floor and wall planes are "detected" shortly after the +/// session opens, mimicking real AR startup. The AR view renders the room +/// plus every anchored `ARNode` through the port's software 3D device, with +/// the virtual device camera driven by the mouse (drag to look) and keyboard +/// (WASD / arrow keys to move, Q / E for up / down). +/// +/// The full core event pipeline - plane events, anchors, hit tests, image +/// and face anchors, light estimates - flows through the same +/// `ARImpl.EventSink` contract the native backends use, so application AR +/// logic is debuggable in the simulator with breakpoints. The +/// Simulate - AR Simulation window (`ARSimulation`) drives error states, +/// lighting, image detection and face toggling. +public class JavaSEARImpl extends ARImpl { + + /// The floor of the virtual room sits 1.4 meters below the session + /// origin, roughly a phone held at chest height. + static final float FLOOR_Y = -1.4f; + private static final float FLOOR_EXTENT = 6f; + private static final float WALL_Z = -3f; + private static final float WALL_HEIGHT = 3f; + private static final float FOV_Y_DEGREES = 60f; + + /// The impl behind the currently open session, reachable by the + /// Simulate menu. Null when no AR session is open. + static volatile JavaSEARImpl activeInstance; + + private EventSink sink; + private ARSessionOptions options; + private volatile boolean closed; + private volatile boolean paused; + private java.util.Timer timer; + + private final Object sceneLock = new Object(); + private float camX; + private float camY; + private float camZ; + private float camYaw; + private float camPitch; + + private final List planes = new ArrayList(); + private final LinkedHashMap anchors = new LinkedHashMap(); + private int anchorSeq; + private int planeSeq; + private String faceAnchorId; + + private ARSurface surface; + + private static final class SimAnchor { + final String id; + ARPose pose; + ARNode node; + + SimAnchor(String id, ARPose pose) { + this.id = id; + this.pose = pose; + } + } + + @Override + public ARCapabilities getCapabilities() { + return new ARCapabilities(true, true, true, true, true); + } + + @Override + public void setEventSink(EventSink sink) { + this.sink = sink; + } + + @Override + public void open(ARSessionOptions opts) { + this.options = opts == null ? new ARSessionOptions() : opts; + activeInstance = this; + timer = new java.util.Timer("ar-simulation", true); + schedule(200, new Runnable() { + public void run() { + sink.onTrackingStateChanged(ARTrackingState.LIMITED, + ARTrackingFailureReason.INITIALIZING); + } + }); + schedule(1000, new Runnable() { + public void run() { + sink.onTrackingStateChanged(ARTrackingState.TRACKING, + ARTrackingFailureReason.NONE); + sink.onLightEstimate(new ARLightEstimate(true, 1f, 1f, 1f, 1f)); + publishCameraPose(); + } + }); + if (options.getTrackingMode() == ARTrackingMode.FACE) { + schedule(1200, new Runnable() { + public void run() { + simSetFaceVisible(true); + } + }); + } else { + if (options.getPlaneDetection().includesHorizontal()) { + schedule(1500, new Runnable() { + public void run() { + detectFloor(); + } + }); + } + if (options.getPlaneDetection().includesVertical()) { + schedule(2200, new Runnable() { + public void run() { + detectWall(); + } + }); + } + } + } + + private void schedule(long delayMillis, final Runnable r) { + java.util.Timer t = timer; + if (t == null) { + return; + } + t.schedule(new TimerTask() { + @Override public void run() { + if (!closed && !paused) { + try { + r.run(); + } catch (Throwable err) { + err.printStackTrace(); + } + } + } + }, delayMillis); + } + + private void detectFloor() { + ARPlane floor = new ARPlane("sim-plane-floor-" + (++planeSeq), + ARPlane.Type.HORIZONTAL_UP, + new ARPose(0f, FLOOR_Y, 0f, 0f, 0f, 0f, 1f), + FLOOR_EXTENT, FLOOR_EXTENT, null, ARTrackingState.TRACKING); + synchronized (sceneLock) { + planes.add(floor); + } + sink.onPlaneAdded(floor); + } + + private void detectWall() { + // The wall faces the viewer: its normal is world +Z, so the plane's + // local Y axis (the normal) is rotated onto +Z by a 90 degree + // rotation around X. + float s = (float) Math.sin(Math.PI / 4); + float c = (float) Math.cos(Math.PI / 4); + ARPlane wall = new ARPlane("sim-plane-wall-" + (++planeSeq), + ARPlane.Type.VERTICAL, + new ARPose(0f, FLOOR_Y + WALL_HEIGHT / 2f, WALL_Z, s, 0f, 0f, c), + FLOOR_EXTENT, WALL_HEIGHT, null, ARTrackingState.TRACKING); + synchronized (sceneLock) { + planes.add(wall); + } + sink.onPlaneAdded(wall); + } + + @Override + public PeerComponent createViewPeer() { + surface = new ARSurface(); + return PeerComponent.create(surface); + } + + @Override + public void hitTest(float xNorm, float yNorm, final AsyncResource result) { + final ARHitResult[] hits = computeHits(xNorm, yNorm); + Display.getInstance().callSerially(new Runnable() { + public void run() { + result.complete(hits); + } + }); + } + + private ARHitResult[] computeHits(float xNorm, float yNorm) { + float ox; + float oy; + float oz; + float[] dir = new float[3]; + ARPlane[] planeSnapshot; + synchronized (sceneLock) { + ox = camX; + oy = camY; + oz = camZ; + // Camera-space ray through the normalized view point. + float aspect = surface == null || surface.getHeight() == 0 ? 1.5f + : (float) surface.getWidth() / (float) surface.getHeight(); + float tanHalf = (float) Math.tan(Math.toRadians(FOV_Y_DEGREES / 2)); + dir[0] = (xNorm * 2f - 1f) * tanHalf * aspect; + dir[1] = (1f - yNorm * 2f) * tanHalf; + dir[2] = -1f; + float[] q = orientationQuat(); + Quaternion.rotateVector(q, dir); + planeSnapshot = planes.toArray(new ARPlane[planes.size()]); + } + float dlen = (float) Math.sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2]); + dir[0] /= dlen; + dir[1] /= dlen; + dir[2] /= dlen; + + List hits = new ArrayList(); + for (ARPlane plane : planeSnapshot) { + ARPose center = plane.getCenterPose(); + if (plane.getType() == ARPlane.Type.VERTICAL) { + // Ray vs the z = WALL_Z plane. + if (Math.abs(dir[2]) < 1e-6f) { + continue; + } + float t = (center.getTz() - oz) / dir[2]; + if (t <= 0) { + continue; + } + float px = ox + dir[0] * t; + float py = oy + dir[1] * t; + if (Math.abs(px - center.getTx()) > plane.getExtentX() / 2 + || Math.abs(py - center.getTy()) > plane.getExtentZ() / 2) { + continue; + } + hits.add(new ARHitResult(new ARPose(px, py, center.getTz(), + center.getQx(), center.getQy(), center.getQz(), center.getQw()), + t, ARHitResult.Type.PLANE, plane, null)); + } else { + // Ray vs the y = FLOOR_Y plane. + if (Math.abs(dir[1]) < 1e-6f) { + continue; + } + float t = (center.getTy() - oy) / dir[1]; + if (t <= 0) { + continue; + } + float px = ox + dir[0] * t; + float pz = oz + dir[2] * t; + if (Math.abs(px - center.getTx()) > plane.getExtentX() / 2 + || Math.abs(pz - center.getTz()) > plane.getExtentZ() / 2) { + continue; + } + hits.add(new ARHitResult(new ARPose(px, center.getTy(), pz, 0f, 0f, 0f, 1f), + t, ARHitResult.Type.PLANE, plane, null)); + } + } + // Nearest first. + for (int i = 0; i < hits.size(); i++) { + for (int j = i + 1; j < hits.size(); j++) { + if (hits.get(j).getDistance() < hits.get(i).getDistance()) { + ARHitResult tmp = hits.get(i); + hits.set(i, hits.get(j)); + hits.set(j, tmp); + } + } + } + return hits.toArray(new ARHitResult[hits.size()]); + } + + @Override + public String createAnchor(ARPose pose) { + String id = "sim-anchor-" + (++anchorSeq); + synchronized (sceneLock) { + anchors.put(id, new SimAnchor(id, pose)); + } + repaintSurface(); + return id; + } + + @Override + public String createAnchorFromHit(Object nativeHandle, ARPose pose) { + return createAnchor(pose); + } + + @Override + public void removeAnchor(String anchorId) { + synchronized (sceneLock) { + anchors.remove(anchorId); + } + repaintSurface(); + } + + @Override + public void setAnchorNode(String anchorId, ARNode node) { + synchronized (sceneLock) { + SimAnchor a = anchors.get(anchorId); + if (a != null) { + a.node = node; + } + } + repaintSurface(); + } + + @Override + public void nodeChanged(String anchorId, ARNode node) { + repaintSurface(); + } + + @Override + public void requestPermissions(final AsyncResource result) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + result.complete(Boolean.TRUE); + } + }); + } + + @Override + public void pause() { + paused = true; + } + + @Override + public void resume() { + paused = false; + publishCameraPose(); + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (timer != null) { + timer.cancel(); + timer = null; + } + if (surface != null) { + surface.stopTimer(); + } + if (activeInstance == this) { + activeInstance = null; + } + } + + // ------------------------------------------------------------------ + // Hooks for the Simulate > AR Simulation window + // ------------------------------------------------------------------ + + /// The session configuration, for the simulation window's status display. + ARSessionOptions getOpenOptions() { + return options; + } + + void simSetTrackingState(ARTrackingState state, ARTrackingFailureReason reason) { + if (!closed) { + sink.onTrackingStateChanged(state, reason); + } + } + + void simSetLight(float intensity, float r, float g, float b) { + if (!closed) { + sink.onLightEstimate(new ARLightEstimate(true, intensity, r, g, b)); + } + } + + String[] simGetReferenceImageNames() { + ARReferenceImage[] imgs = options == null + ? new ARReferenceImage[0] : options.getReferenceImages(); + String[] names = new String[imgs.length]; + for (int i = 0; i < imgs.length; i++) { + names[i] = imgs[i].getName(); + } + return names; + } + + /// Simulates the camera recognizing a registered reference image one + /// meter in front of the current virtual camera. + void simDetectImage(String name) { + if (closed) { + return; + } + float physicalWidth = 0.3f; + ARReferenceImage[] imgs = options.getReferenceImages(); + for (ARReferenceImage img : imgs) { + if (img.getName().equals(name)) { + physicalWidth = img.getPhysicalWidthMeters(); + } + } + ARPose pose = poseInFrontOfCamera(1f); + String id = "sim-image-" + (++anchorSeq); + synchronized (sceneLock) { + anchors.put(id, new SimAnchor(id, pose)); + } + sink.onAnchorAdded(new ARImageAnchor(id, pose, name, physicalWidth)); + repaintSurface(); + } + + /// Adds or removes the simulated face anchor (FACE tracking mode). + void simSetFaceVisible(boolean visible) { + if (closed) { + return; + } + if (visible) { + if (faceAnchorId != null) { + return; + } + ARPose pose = poseInFrontOfCamera(0.4f); + faceAnchorId = "sim-face-" + (++anchorSeq); + synchronized (sceneLock) { + anchors.put(faceAnchorId, new SimAnchor(faceAnchorId, pose)); + } + ARFaceAnchor face = new ARFaceAnchor(faceAnchorId, pose); + sink.onAnchorAdded(face); + // Ship a basic geometry payload so getRegionPose()/mesh code paths + // are exercised in the simulator. + ARPose[] regions = new ARPose[ARFaceRegion.values().length]; + regions[ARFaceRegion.NOSE_TIP.ordinal()] = pose.transform( + new ARPose(0f, 0f, 0.06f, 0f, 0f, 0f, 1f)); + regions[ARFaceRegion.FOREHEAD_LEFT.ordinal()] = pose.transform( + new ARPose(-0.05f, 0.08f, 0.02f, 0f, 0f, 0f, 1f)); + regions[ARFaceRegion.FOREHEAD_RIGHT.ordinal()] = pose.transform( + new ARPose(0.05f, 0.08f, 0.02f, 0f, 0f, 0f, 1f)); + float[] meshVertices = { + -0.08f, -0.1f, 0f, + 0.08f, -0.1f, 0f, + 0.08f, 0.1f, 0f, + -0.08f, 0.1f, 0f + }; + int[] meshTriangles = {0, 1, 2, 0, 2, 3}; + sink.onFaceAnchorUpdated(faceAnchorId, pose, ARTrackingState.TRACKING, + regions, meshVertices, meshTriangles); + } else { + if (faceAnchorId == null) { + return; + } + String id = faceAnchorId; + faceAnchorId = null; + synchronized (sceneLock) { + anchors.remove(id); + } + sink.onAnchorRemoved(id); + } + repaintSurface(); + } + + boolean simIsFaceVisible() { + return faceAnchorId != null; + } + + /// Clears and re-runs the plane detection sequence. + void simRedetectPlanes() { + if (closed || options.getTrackingMode() == ARTrackingMode.FACE) { + return; + } + ARPlane[] old; + synchronized (sceneLock) { + old = planes.toArray(new ARPlane[planes.size()]); + planes.clear(); + } + for (ARPlane p : old) { + sink.onPlaneRemoved(p.getId()); + } + if (options.getPlaneDetection().includesHorizontal()) { + schedule(600, new Runnable() { + public void run() { + detectFloor(); + } + }); + } + if (options.getPlaneDetection().includesVertical()) { + schedule(1200, new Runnable() { + public void run() { + detectWall(); + } + }); + } + repaintSurface(); + } + + // ------------------------------------------------------------------ + // Virtual camera + // ------------------------------------------------------------------ + + private float[] orientationQuat() { + float[] yawQ = Quaternion.fromAxisAngle(-camYaw, 0f, 1f, 0f); + float[] pitchQ = Quaternion.fromAxisAngle(camPitch, 1f, 0f, 0f); + float[] q = new float[4]; + Quaternion.multiply(yawQ, pitchQ, q); + return q; + } + + private ARPose cameraPose() { + float[] q; + float x; + float y; + float z; + synchronized (sceneLock) { + q = orientationQuat(); + x = camX; + y = camY; + z = camZ; + } + return new ARPose(x, y, z, q[0], q[1], q[2], q[3]); + } + + private ARPose poseInFrontOfCamera(float distanceMeters) { + return cameraPose().transform( + new ARPose(0f, 0f, -distanceMeters, 0f, 0f, 0f, 1f)); + } + + private void publishCameraPose() { + if (!closed && !paused && sink != null) { + sink.onCameraPose(cameraPose()); + } + } + + private void repaintSurface() { + ARSurface s = surface; + if (s != null) { + s.repaint(); + } + } + + // ------------------------------------------------------------------ + // The Swing rendering surface + // ------------------------------------------------------------------ + + /// Renders the virtual room and anchored content through the port's + /// software 3D device, and owns the mouse / keyboard camera controls. + private final class ARSurface extends JComponent { + private final JavaSESoftwareDevice device = new JavaSESoftwareDevice(); + private final Camera camera = new Camera(); + private final Light light = new Light(); + private final IdentityHashMap materials = + new IdentityHashMap(); + private Mesh floorMesh; + private Mesh wallMesh; + private Mesh anchorMarker; + private Material floorMaterial; + private Material wallMaterial; + private Material markerMaterial; + private boolean initialized; + private int lastW = -1; + private int lastH = -1; + private final javax.swing.Timer repaintTimer; + private int lastMouseX; + private int lastMouseY; + + ARSurface() { + setOpaque(true); + setFocusable(true); + // A gentle continuous repaint keeps the camera-pose publishing + // and view fresh, mirroring a live camera feed. + repaintTimer = new javax.swing.Timer(50, new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent e) { + repaint(); + } + }); + repaintTimer.start(); + MouseAdapter mouse = new MouseAdapter() { + @Override public void mousePressed(MouseEvent e) { + lastMouseX = e.getX(); + lastMouseY = e.getY(); + requestFocusInWindow(); + } + + @Override public void mouseDragged(MouseEvent e) { + synchronized (sceneLock) { + camYaw += (e.getX() - lastMouseX) * 0.006f; + camPitch -= (e.getY() - lastMouseY) * 0.006f; + float limit = (float) Math.toRadians(89); + if (camPitch > limit) { + camPitch = limit; + } + if (camPitch < -limit) { + camPitch = -limit; + } + } + lastMouseX = e.getX(); + lastMouseY = e.getY(); + publishCameraPose(); + } + }; + addMouseListener(mouse); + addMouseMotionListener(mouse); + addKeyListener(new KeyAdapter() { + @Override public void keyPressed(KeyEvent e) { + float step = 0.08f; + float fx = 0; + float fz = 0; + float dy = 0; + switch (e.getKeyCode()) { + case KeyEvent.VK_W: + case KeyEvent.VK_UP: + fz = -step; + break; + case KeyEvent.VK_S: + case KeyEvent.VK_DOWN: + fz = step; + break; + case KeyEvent.VK_A: + case KeyEvent.VK_LEFT: + fx = -step; + break; + case KeyEvent.VK_D: + case KeyEvent.VK_RIGHT: + fx = step; + break; + case KeyEvent.VK_Q: + dy = step; + break; + case KeyEvent.VK_E: + dy = -step; + break; + default: + return; + } + synchronized (sceneLock) { + float sin = (float) Math.sin(camYaw); + float cos = (float) Math.cos(camYaw); + // Move in the yaw-rotated horizontal frame. + camX += fx * cos - fz * sin; + camZ += fx * sin + fz * cos; + camY += dy; + } + publishCameraPose(); + } + }); + } + + void stopTimer() { + repaintTimer.stop(); + } + + @Override + public void removeNotify() { + super.removeNotify(); + repaintTimer.stop(); + } + + @Override + public void addNotify() { + super.addNotify(); + if (!closed) { + repaintTimer.start(); + } + } + + @Override + protected void paintComponent(Graphics g) { + int w = getWidth(); + int h = getHeight(); + if (w <= 0 || h <= 0) { + return; + } + device.resize(w, h); + if (!initialized) { + initScene(); + initialized = true; + } + if (w != lastW || h != lastH) { + lastW = w; + lastH = h; + device.setViewport(0, 0, w, h); + camera.setAspect((float) w / (float) h); + } + renderScene(); + BufferedImage img = device.getImage(); + if (img != null) { + g.drawImage(img, 0, 0, null); + } + } + + private void initScene() { + // The software rasterizer rejects triangles that cross the near + // plane instead of clipping them, so the room surfaces must be + // tessellated: only the cells at the viewer's feet drop out + // rather than the whole plane. + floorMesh = gridMesh(FLOOR_EXTENT, FLOOR_EXTENT, 16, 16); + wallMesh = gridMesh(FLOOR_EXTENT, WALL_HEIGHT, 16, 8); + anchorMarker = Primitives.cube(device, 0.06f); + floorMaterial = new Material(Material.Type.UNLIT) + .setTexture(createGridTexture(0xff3c78b4, 0xff2d5a87)); + wallMaterial = new Material(Material.Type.UNLIT) + .setTexture(createGridTexture(0xff6a6a72, 0xff55555c)); + markerMaterial = new Material(Material.Type.UNLIT).setColor(0xffffc020); + light.setDirection(-0.4f, -1f, -0.3f); + light.setColor(0xffffffff); + camera.setPerspective(FOV_Y_DEGREES, 0.05f, 100f); + } + + /// Builds a subdivided quad in the XY plane facing +Z, centered at the + /// origin, with 0..1 texture coordinates - Primitives.quad with + /// tessellation. + private Mesh gridMesh(float width, float height, int cols, int rows) { + int vertexCount = (cols + 1) * (rows + 1); + float[] v = new float[vertexCount * 8]; + int o = 0; + for (int r = 0; r <= rows; r++) { + float fy = (float) r / rows; + for (int c = 0; c <= cols; c++) { + float fx = (float) c / cols; + v[o] = (fx - 0.5f) * width; + v[o + 1] = (fy - 0.5f) * height; + v[o + 2] = 0f; + v[o + 3] = 0f; + v[o + 4] = 0f; + v[o + 5] = 1f; + v[o + 6] = fx; + v[o + 7] = 1f - fy; + o += 8; + } + } + int[] idx = new int[cols * rows * 6]; + int i = 0; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + int first = r * (cols + 1) + c; + int second = first + cols + 1; + idx[i] = first; + idx[i + 1] = first + 1; + idx[i + 2] = second + 1; + idx[i + 3] = first; + idx[i + 4] = second + 1; + idx[i + 5] = second; + i += 6; + } + } + com.codename1.gpu.VertexBuffer vb = device.createVertexBuffer( + com.codename1.gpu.VertexFormat.POSITION_NORMAL_TEXCOORD, vertexCount); + vb.setData(v); + com.codename1.gpu.IndexBuffer ib = device.createIndexBuffer(idx.length); + ib.setData(idx); + return new Mesh(vb, ib, com.codename1.gpu.PrimitiveType.TRIANGLES); + } + + private Texture createGridTexture(int lineColor, int fillColor) { + int size = 256; + int cell = 32; + int[] argb = new int[size * size]; + for (int y = 0; y < size; y++) { + for (int x = 0; x < size; x++) { + boolean line = x % cell == 0 || y % cell == 0; + argb[y * size + x] = line ? lineColor : fillColor; + } + } + Texture t = device.createTexture(size, size, argb); + t.setFilter(Texture.Filter.LINEAR); + return t; + } + + private void renderScene() { + float[] q; + float ex; + float ey; + float ez; + ARPlane[] planeSnapshot; + Object[][] anchorSnapshot; + synchronized (sceneLock) { + q = orientationQuat(); + ex = camX; + ey = camY; + ez = camZ; + planeSnapshot = planes.toArray(new ARPlane[planes.size()]); + anchorSnapshot = new Object[anchors.size()][]; + int i = 0; + for (SimAnchor a : anchors.values()) { + anchorSnapshot[i++] = new Object[]{a.pose, a.node}; + } + } + float[] fwd = {0f, 0f, -1f}; + Quaternion.rotateVector(q, fwd); + float[] up = {0f, 1f, 0f}; + Quaternion.rotateVector(q, up); + camera.setPosition(ex, ey, ez); + camera.setTarget(ex + fwd[0], ey + fwd[1], ez + fwd[2]); + camera.setUp(up[0], up[1], up[2]); + + device.clear(0xff26303a, true, true); + device.setCamera(camera); + device.setLight(light); + + for (ARPlane plane : planeSnapshot) { + if (plane.getType() == ARPlane.Type.VERTICAL) { + // The tessellated wall quad already faces +Z which is the + // wall normal. + float[] m = Matrix4.translation(plane.getCenterPose().getTx(), + plane.getCenterPose().getTy(), plane.getCenterPose().getTz()); + device.draw(wallMesh, wallMaterial, m); + } else { + // Lay the +Z-facing quad flat (normal up) on the floor. + float[] m = new float[16]; + Matrix4.multiply(Matrix4.translation(plane.getCenterPose().getTx(), + plane.getCenterPose().getTy(), plane.getCenterPose().getTz()), + Matrix4.rotation((float) (-Math.PI / 2), 1f, 0f, 0f), m); + device.draw(floorMesh, floorMaterial, m); + } + } + + for (Object[] entry : anchorSnapshot) { + ARPose pose = (ARPose) entry[0]; + ARNode node = (ARNode) entry[1]; + float[] anchorMatrix = pose.toMatrix(); + device.draw(anchorMarker, markerMaterial, anchorMatrix); + if (node != null) { + drawNode(anchorMatrix, node); + } + } + } + + private void drawNode(float[] parentMatrix, ARNode node) { + if (!node.isVisible()) { + return; + } + float[] local = Matrix4.translation(node.getLocalX(), node.getLocalY(), + node.getLocalZ()); + float[] rot = new float[16]; + Quaternion.toMatrix(new float[]{node.getLocalQx(), node.getLocalQy(), + node.getLocalQz(), node.getLocalQw()}, rot); + float[] tmp = new float[16]; + Matrix4.multiply(local, rot, tmp); + float s = node.getLocalScale(); + float[] m = new float[16]; + Matrix4.multiply(tmp, Matrix4.scaling(s, s, s), m); + float[] world = new float[16]; + Matrix4.multiply(parentMatrix, m, world); + + ARModel model = node.getModel(); + if (model != null) { + Mesh mesh = model.getMesh(); + if (mesh != null) { + device.draw(mesh, materialFor(model), world); + } + } + for (int i = 0; i < node.getChildCount(); i++) { + drawNode(world, node.getChildAt(i)); + } + } + + private Material materialFor(ARModel model) { + Material m = materials.get(model); + if (m == null) { + com.codename1.ui.Image img = model.getBaseColorImage(); + if (img != null) { + Texture t = device.createTexture(img); + t.setFilter(Texture.Filter.LINEAR); + m = new Material(Material.Type.PHONG).setTexture(t); + } else { + m = new Material(Material.Type.PHONG).setColor(model.getColor()); + } + materials.put(model, m); + } + return m; + } + } +} diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java index 5b7033dd62b..fb19cc63458 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEPort.java @@ -882,6 +882,7 @@ public static void setShowEDTViolationStacks(boolean aShowEDTViolationStacks) { private static PerformanceMonitor perfMonitor; static LocationSimulation locSimulation; static MotionSimulation motionSimulation; + static ARSimulation arSimulation; private JavaSEMotionSensorManager motionSensorManager; static PushSimulator pushSimulation; private static boolean blockMonitors; @@ -5794,6 +5795,18 @@ public void actionPerformed(ActionEvent ae) { }); simulateMenu.add(motionSim); + JMenuItem arSim = new JMenuItem("AR Simulation"); + arSim.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent ae) { + if(arSimulation == null) { + arSimulation = new ARSimulation(); + } + arSimulation.setVisible(true); + } + }); + simulateMenu.add(arSim); + JMenuItem pushSim = new JMenuItem("Push Simulation"); pushSim.addActionListener(new ActionListener() { @Override @@ -14542,6 +14555,11 @@ public void capturePhoto(final com.codename1.ui.events.ActionListener response) public com.codename1.impl.CameraImpl createCameraImpl() { return new JavaSECameraImpl(); } + + @Override + public com.codename1.impl.ARImpl createARImpl() { + return new JavaSEARImpl(); + } private void captureMulti(final com.codename1.ui.events.ActionListener response, final String[] imageTypes, final String desc) { SwingUtilities.invokeLater(new Runnable() { diff --git a/Ports/iOSPort/nativeSources/CN1AR.h b/Ports/iOSPort/nativeSources/CN1AR.h new file mode 100644 index 00000000000..23572adc302 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1AR.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +// Native side of the com.codename1.ar API: an ARKit ARSession composited +// through an ARSCNView (SceneKit renders the camera background, syncs anchor +// transforms and applies the light estimate automatically). Anchored content +// arrives from Java as raw interleaved vertex buffers (parsed from glTF or +// built by com.codename1.gpu) and becomes SCNGeometry. +// +// The whole class is gated on INCLUDE_CN1_AR (uncommented by IPhoneBuilder +// only for apps that reference com.codename1.ar) and compiled out on +// tvOS/watchOS where ARKit does not exist. The IOSNative bridge symbols in +// CN1AR.m exist unconditionally so linking always succeeds. + +#import "CodenameOne_GLViewController.h" + +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + +#import +#import +#import + +@interface CN1AR : NSObject + +@property (nonatomic, strong) ARSCNView *sceneView; +@property (nonatomic, strong) NSMutableArray *referenceImages; +// anchorId (UUID string) -> the SCNNode ARSCNView created for that anchor. +@property (nonatomic, strong) NSMutableDictionary *anchorNodes; +// anchorId -> the content container node built from Java meshes. +@property (nonatomic, strong) NSMutableDictionary *contentNodes; +// Recent raycast results kept alive so createAnchorFromHit can anchor to the +// exact native hit. Cleared on every new hit test. +@property (nonatomic, strong) NSMutableDictionary *recentHits; +@property (nonatomic, assign) int nextHitId; +@property (nonatomic, assign) int configType; +@property (nonatomic, assign) int planeMask; +@property (nonatomic, assign) BOOL lightEstimation; +@property (nonatomic, assign) int frameCounter; +@property (nonatomic, assign) BOOL closed; + +- (void)addReferenceImage:(NSData *)encoded name:(NSString *)name width:(float)widthMeters; +- (BOOL)startWithConfigType:(int)configType planeMask:(int)planeMask + lightEstimation:(BOOL)lightEstimation; +- (UIView *)createView; +- (NSString *)hitTestX:(float)xNorm y:(float)yNorm; +- (NSString *)createAnchorTx:(float)tx ty:(float)ty tz:(float)tz + qx:(float)qx qy:(float)qy qz:(float)qz qw:(float)qw; +- (NSString *)createAnchorFromHit:(int)hitId; +- (void)removeAnchorById:(NSString *)anchorId; +- (void)clearAnchorContent:(NSString *)anchorId; +- (void)addAnchorMesh:(NSString *)anchorId + interleaved:(const float *)interleaved + vertexCount:(int)vertexCount + indices:(const int *)indices + indexCount:(int)indexCount + argb:(int)argb + encodedTexture:(NSData *)encodedTexture + localTransform:(const float *)localTransform16; +- (void)pauseSession; +- (void)resumeSession; +- (void)closeSession; + +@end + +#endif // INCLUDE_CN1_AR diff --git a/Ports/iOSPort/nativeSources/CN1AR.m b/Ports/iOSPort/nativeSources/CN1AR.m new file mode 100644 index 00000000000..92438791e34 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1AR.m @@ -0,0 +1,777 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +#import "CN1AR.h" +#import "xmlvm.h" + +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH +#import +#import "java_lang_String.h" +#import "com_codename1_impl_ios_IOSARImpl.h" + +// Tracking state / failure reason codes shared with IOSARImpl.java. +#define CN1AR_STATE_NOT_TRACKING 0 +#define CN1AR_STATE_LIMITED 1 +#define CN1AR_STATE_TRACKING 2 +#define CN1AR_REASON_NONE 0 +#define CN1AR_REASON_INITIALIZING 1 +#define CN1AR_REASON_EXCESSIVE_MOTION 2 +#define CN1AR_REASON_INSUFFICIENT_LIGHT 3 +#define CN1AR_REASON_INSUFFICIENT_FEATURES 4 + +// Extracts translation + rotation quaternion from a rigid transform into a +// 7-float pose (tx ty tz qx qy qz qw). +static void cn1arPose(simd_float4x4 m, float *out7) { + out7[0] = m.columns[3].x; + out7[1] = m.columns[3].y; + out7[2] = m.columns[3].z; + simd_quatf q = simd_quaternion(m); + out7[3] = q.vector.x; + out7[4] = q.vector.y; + out7[5] = q.vector.z; + out7[6] = q.vector.w; +} + +static void cn1arRunOnMain(void (^block)(void)) { + if ([NSThread isMainThread]) { + block(); + } else { + dispatch_sync(dispatch_get_main_queue(), block); + } +} + +@implementation CN1AR + +- (instancetype)init { + self = [super init]; + if (self) { + _referenceImages = [NSMutableArray array]; + _anchorNodes = [NSMutableDictionary dictionary]; + _contentNodes = [NSMutableDictionary dictionary]; + _recentHits = [NSMutableDictionary dictionary]; + _nextHitId = 1; + cn1arRunOnMain(^{ + self.sceneView = [[ARSCNView alloc] initWithFrame:CGRectMake(0, 0, 320, 480)]; + self.sceneView.delegate = self; + self.sceneView.session.delegate = self; + self.sceneView.automaticallyUpdatesLighting = YES; + self.sceneView.autoenablesDefaultLighting = YES; + }); + } + return self; +} + ++ (BOOL)isConfigSupported:(int)configType { + if (configType == 1) { + return [ARFaceTrackingConfiguration isSupported]; + } + return [ARWorldTrackingConfiguration isSupported]; +} + +- (void)addReferenceImage:(NSData *)encoded name:(NSString *)name width:(float)widthMeters { + UIImage *img = [UIImage imageWithData:encoded]; + if (img == nil || img.CGImage == nil) { + return; + } + ARReferenceImage *ref = [[ARReferenceImage alloc] + initWithCGImage:img.CGImage + orientation:kCGImagePropertyOrientationUp + physicalWidth:widthMeters]; + ref.name = name; + [self.referenceImages addObject:ref]; +} + +- (BOOL)startWithConfigType:(int)configType planeMask:(int)planeMask + lightEstimation:(BOOL)lightEstimation { + if (![CN1AR isConfigSupported:configType]) { + return NO; + } + self.configType = configType; + self.planeMask = planeMask; + self.lightEstimation = lightEstimation; + cn1arRunOnMain(^{ + [self runConfigurationWithReset:YES]; + }); + return YES; +} + +- (void)runConfigurationWithReset:(BOOL)reset { + ARConfiguration *config; + if (self.configType == 1) { + ARFaceTrackingConfiguration *face = [[ARFaceTrackingConfiguration alloc] init]; + config = face; + } else { + ARWorldTrackingConfiguration *world = [[ARWorldTrackingConfiguration alloc] init]; + ARPlaneDetection detection = ARPlaneDetectionNone; + if (self.planeMask & 1) { + detection |= ARPlaneDetectionHorizontal; + } + if (self.planeMask & 2) { + detection |= ARPlaneDetectionVertical; + } + world.planeDetection = detection; + if (self.referenceImages.count > 0) { + world.detectionImages = [NSSet setWithArray:self.referenceImages]; + if (@available(iOS 12.0, *)) { + world.maximumNumberOfTrackedImages = (NSInteger) self.referenceImages.count; + } + } + config = world; + } + config.lightEstimationEnabled = self.lightEstimation; + ARSessionRunOptions options = reset + ? (ARSessionRunOptionResetTracking | ARSessionRunOptionRemoveExistingAnchors) + : 0; + [self.sceneView.session runWithConfiguration:config options:options]; +} + +- (UIView *)createView { + return self.sceneView; +} + +- (NSString *)hitTestX:(float)xNorm y:(float)yNorm { + NSMutableString * __block out = [NSMutableString string]; + cn1arRunOnMain(^{ + [self.recentHits removeAllObjects]; + CGSize size = self.sceneView.bounds.size; + CGPoint point = CGPointMake(xNorm * size.width, yNorm * size.height); + NSArray *results = @[]; + if (@available(iOS 13.0, *)) { + ARRaycastQuery *query = [self.sceneView + raycastQueryFromPoint:point + allowingTarget:ARRaycastTargetExistingPlaneGeometry + alignment:ARRaycastTargetAlignmentAny]; + if (query != nil) { + results = [self.sceneView.session raycast:query]; + } + if (results.count == 0) { + query = [self.sceneView + raycastQueryFromPoint:point + allowingTarget:ARRaycastTargetEstimatedPlane + alignment:ARRaycastTargetAlignmentAny]; + if (query != nil) { + results = [self.sceneView.session raycast:query]; + } + } + } + simd_float4x4 camTransform = self.sceneView.session.currentFrame != nil + ? self.sceneView.session.currentFrame.camera.transform + : matrix_identity_float4x4; + simd_float3 camPos = simd_make_float3(camTransform.columns[3].x, + camTransform.columns[3].y, camTransform.columns[3].z); + for (ARRaycastResult *r in results) { + int hitId = self.nextHitId++; + self.recentHits[@(hitId)] = r; + float pose[7]; + cn1arPose(r.worldTransform, pose); + simd_float3 hitPos = simd_make_float3(pose[0], pose[1], pose[2]); + float distance = simd_distance(camPos, hitPos); + int type = r.target == ARRaycastTargetExistingPlaneGeometry ? 0 : 1; + NSString *planeId = @""; + if ([r.anchor isKindOfClass:[ARPlaneAnchor class]]) { + planeId = r.anchor.identifier.UUIDString; + type = 0; + } + if (out.length > 0) { + [out appendString:@";"]; + } + [out appendFormat:@"%d|%d|%f|%f|%f|%f|%f|%f|%f|%f|%@", + hitId, type, pose[0], pose[1], pose[2], + pose[3], pose[4], pose[5], pose[6], distance, planeId]; + } + }); + return out; +} + +- (NSString *)createAnchorTx:(float)tx ty:(float)ty tz:(float)tz + qx:(float)qx qy:(float)qy qz:(float)qz qw:(float)qw { + simd_quatf q = simd_quaternion(qx, qy, qz, qw); + simd_float4x4 m = simd_matrix4x4(q); + m.columns[3] = simd_make_float4(tx, ty, tz, 1); + ARAnchor *anchor = [[ARAnchor alloc] initWithTransform:m]; + NSString * __block anchorId = anchor.identifier.UUIDString; + cn1arRunOnMain(^{ + [self.sceneView.session addAnchor:anchor]; + }); + return anchorId; +} + +- (NSString *)createAnchorFromHit:(int)hitId { + ARRaycastResult * __block result = nil; + NSString * __block anchorId = nil; + cn1arRunOnMain(^{ + result = self.recentHits[@(hitId)]; + if (result != nil) { + ARAnchor *anchor = [[ARAnchor alloc] initWithTransform:result.worldTransform]; + anchorId = anchor.identifier.UUIDString; + [self.sceneView.session addAnchor:anchor]; + } + }); + return anchorId; +} + +- (ARAnchor *)findAnchorById:(NSString *)anchorId { + ARFrame *frame = self.sceneView.session.currentFrame; + for (ARAnchor *a in frame.anchors) { + if ([a.identifier.UUIDString isEqualToString:anchorId]) { + return a; + } + } + return nil; +} + +- (void)removeAnchorById:(NSString *)anchorId { + cn1arRunOnMain(^{ + ARAnchor *a = [self findAnchorById:anchorId]; + if (a != nil) { + [self.sceneView.session removeAnchor:a]; + } + [self.contentNodes removeObjectForKey:anchorId]; + [self.anchorNodes removeObjectForKey:anchorId]; + }); +} + +- (void)clearAnchorContent:(NSString *)anchorId { + cn1arRunOnMain(^{ + SCNNode *content = self.contentNodes[anchorId]; + if (content != nil) { + [content removeFromParentNode]; + [self.contentNodes removeObjectForKey:anchorId]; + } + }); +} + +- (void)addAnchorMesh:(NSString *)anchorId + interleaved:(const float *)interleaved + vertexCount:(int)vertexCount + indices:(const int *)indices + indexCount:(int)indexCount + argb:(int)argb + encodedTexture:(NSData *)encodedTexture + localTransform:(const float *)localTransform16 { + // Interleaved layout: position(3) + normal(3) + texcoord(2), 32-byte + // stride -- the VertexFormat.POSITION_NORMAL_TEXCOORD convention every + // com.codename1.gpu mesh producer follows. + int strideBytes = 8 * sizeof(float); + NSData *vertexData = [NSData dataWithBytes:interleaved + length:(NSUInteger) vertexCount * strideBytes]; + SCNGeometrySource *positions = [SCNGeometrySource + geometrySourceWithData:vertexData + semantic:SCNGeometrySourceSemanticVertex + vectorCount:vertexCount + floatComponents:YES + componentsPerVector:3 + bytesPerComponent:sizeof(float) + dataOffset:0 + dataStride:strideBytes]; + SCNGeometrySource *normals = [SCNGeometrySource + geometrySourceWithData:vertexData + semantic:SCNGeometrySourceSemanticNormal + vectorCount:vertexCount + floatComponents:YES + componentsPerVector:3 + bytesPerComponent:sizeof(float) + dataOffset:3 * sizeof(float) + dataStride:strideBytes]; + SCNGeometrySource *uvs = [SCNGeometrySource + geometrySourceWithData:vertexData + semantic:SCNGeometrySourceSemanticTexcoord + vectorCount:vertexCount + floatComponents:YES + componentsPerVector:2 + bytesPerComponent:sizeof(float) + dataOffset:6 * sizeof(float) + dataStride:strideBytes]; + NSData *indexData = [NSData dataWithBytes:indices + length:(NSUInteger) indexCount * sizeof(int)]; + SCNGeometryElement *element = [SCNGeometryElement + geometryElementWithData:indexData + primitiveType:SCNGeometryPrimitiveTypeTriangles + primitiveCount:indexCount / 3 + bytesPerIndex:sizeof(int)]; + SCNGeometry *geometry = [SCNGeometry geometryWithSources:@[positions, normals, uvs] + elements:@[element]]; + SCNMaterial *material = [SCNMaterial material]; + material.lightingModelName = SCNLightingModelBlinn; + if (encodedTexture != nil && encodedTexture.length > 0) { + UIImage *tex = [UIImage imageWithData:encodedTexture]; + if (tex != nil) { + material.diffuse.contents = tex; + } + } else { + float a = ((argb >> 24) & 0xff) / 255.0f; + float r = ((argb >> 16) & 0xff) / 255.0f; + float g = ((argb >> 8) & 0xff) / 255.0f; + float b = (argb & 0xff) / 255.0f; + material.diffuse.contents = [UIColor colorWithRed:r green:g blue:b alpha:a]; + } + material.doubleSided = YES; + geometry.materials = @[material]; + + SCNNode *meshNode = [SCNNode nodeWithGeometry:geometry]; + simd_float4x4 local; + for (int c = 0; c < 4; c++) { + local.columns[c] = simd_make_float4(localTransform16[c * 4], + localTransform16[c * 4 + 1], localTransform16[c * 4 + 2], + localTransform16[c * 4 + 3]); + } + meshNode.simdTransform = local; + + cn1arRunOnMain(^{ + SCNNode *content = self.contentNodes[anchorId]; + if (content == nil) { + content = [SCNNode node]; + self.contentNodes[anchorId] = content; + SCNNode *anchorNode = self.anchorNodes[anchorId]; + if (anchorNode != nil) { + [anchorNode addChildNode:content]; + } + // When the ARSCNView has not yet created the node for this + // anchor, the content stays parked in contentNodes and is + // attached from renderer:didAddNode:forAnchor:. + } + [content addChildNode:meshNode]; + }); +} + +- (void)pauseSession { + cn1arRunOnMain(^{ + [self.sceneView.session pause]; + }); +} + +- (void)resumeSession { + cn1arRunOnMain(^{ + [self runConfigurationWithReset:NO]; + }); +} + +- (void)closeSession { + self.closed = YES; + cn1arRunOnMain(^{ + [self.sceneView.session pause]; + self.sceneView.delegate = nil; + self.sceneView.session.delegate = nil; + [self.recentHits removeAllObjects]; + [self.contentNodes removeAllObjects]; + [self.anchorNodes removeAllObjects]; + }); +} + +#pragma mark - Pose helpers + +- (int)planeTypeFor:(ARPlaneAnchor *)plane { + if (plane.alignment == ARPlaneAnchorAlignmentVertical) { + return 2; + } + // ARKit reports a single "horizontal" alignment; distinguish up/down + // facing surfaces (table vs ceiling) from the plane's world Y axis. + simd_float4 yAxis = plane.transform.columns[1]; + return yAxis.y >= 0 ? 0 : 1; +} + +- (void)firePlaneEvent:(int)kind anchor:(ARPlaneAnchor *)plane { + // The plane's center is expressed relative to the anchor transform. + simd_float4x4 centerM = plane.transform; + simd_float4 center = simd_make_float4(plane.center.x, plane.center.y, plane.center.z, 1); + simd_float4 world = simd_mul(centerM, center); + float pose[7]; + cn1arPose(centerM, pose); + pose[0] = world.x; + pose[1] = world.y; + pose[2] = world.z; + float extentX = plane.extent.x; + float extentZ = plane.extent.z; + JAVA_OBJECT planeId = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG plane.identifier.UUIDString); + com_codename1_impl_ios_IOSARImpl_onPlaneEvent___int_java_lang_String_int_float_float_float_float_float_float_float_float_float( + CN1_THREAD_GET_STATE_PASS_ARG kind, planeId, [self planeTypeFor:plane], + pose[0], pose[1], pose[2], pose[3], pose[4], pose[5], pose[6], + extentX, extentZ); +} + +- (void)fireAnchorUpdated:(ARAnchor *)anchor { + float pose[7]; + cn1arPose(anchor.transform, pose); + int state = CN1AR_STATE_TRACKING; + if ([anchor respondsToSelector:@selector(isTracked)]) { + // ARImageAnchor / ARFaceAnchor expose isTracked. + state = ((BOOL) [(id) anchor isTracked]) ? CN1AR_STATE_TRACKING + : CN1AR_STATE_LIMITED; + } + JAVA_OBJECT anchorId = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG anchor.identifier.UUIDString); + com_codename1_impl_ios_IOSARImpl_onAnchorUpdated___java_lang_String_float_float_float_float_float_float_float_int( + CN1_THREAD_GET_STATE_PASS_ARG anchorId, + pose[0], pose[1], pose[2], pose[3], pose[4], pose[5], pose[6], state); +} + +- (void)fireFaceRegions:(ARFaceAnchor *)face { + // World transforms for the eye regions; forehead/nose are not supplied + // by ARKit so the Java side leaves those regions null. + simd_float4x4 left = simd_mul(face.transform, face.leftEyeTransform); + simd_float4x4 right = simd_mul(face.transform, face.rightEyeTransform); + float lp[7]; + float rp[7]; + cn1arPose(left, lp); + cn1arPose(right, rp); + NSString *packed = [NSString stringWithFormat: + @"%f,%f,%f,%f,%f,%f,%f;%f,%f,%f,%f,%f,%f,%f", + lp[0], lp[1], lp[2], lp[3], lp[4], lp[5], lp[6], + rp[0], rp[1], rp[2], rp[3], rp[4], rp[5], rp[6]]; + JAVA_OBJECT anchorId = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG face.identifier.UUIDString); + JAVA_OBJECT packedJ = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG packed); + com_codename1_impl_ios_IOSARImpl_onFaceRegionsUpdated___java_lang_String_java_lang_String( + CN1_THREAD_GET_STATE_PASS_ARG anchorId, packedJ); +} + +#pragma mark - ARSCNViewDelegate + +- (void)renderer:(id)renderer didAddNode:(SCNNode *)node + forAnchor:(ARAnchor *)anchor { + if (self.closed) { + return; + } + NSString *anchorId = anchor.identifier.UUIDString; + dispatch_async(dispatch_get_main_queue(), ^{ + self.anchorNodes[anchorId] = node; + SCNNode *content = self.contentNodes[anchorId]; + if (content != nil && content.parentNode == nil) { + [node addChildNode:content]; + } + }); + if ([anchor isKindOfClass:[ARPlaneAnchor class]]) { + [self firePlaneEvent:0 anchor:(ARPlaneAnchor *) anchor]; + } else if ([anchor isKindOfClass:[ARImageAnchor class]]) { + ARImageAnchor *img = (ARImageAnchor *) anchor; + float pose[7]; + cn1arPose(img.transform, pose); + JAVA_OBJECT idj = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + JAVA_OBJECT namej = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + img.referenceImage.name == nil ? @"" : img.referenceImage.name); + com_codename1_impl_ios_IOSARImpl_onImageAnchorAdded___java_lang_String_java_lang_String_float_float_float_float_float_float_float_float( + CN1_THREAD_GET_STATE_PASS_ARG idj, namej, + pose[0], pose[1], pose[2], pose[3], pose[4], pose[5], pose[6], + (float) img.referenceImage.physicalSize.width); + } else if ([anchor isKindOfClass:[ARFaceAnchor class]]) { + float pose[7]; + cn1arPose(anchor.transform, pose); + JAVA_OBJECT idj = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + com_codename1_impl_ios_IOSARImpl_onFaceAnchorAdded___java_lang_String_float_float_float_float_float_float_float( + CN1_THREAD_GET_STATE_PASS_ARG idj, + pose[0], pose[1], pose[2], pose[3], pose[4], pose[5], pose[6]); + [self fireFaceRegions:(ARFaceAnchor *) anchor]; + } +} + +- (void)renderer:(id)renderer didUpdateNode:(SCNNode *)node + forAnchor:(ARAnchor *)anchor { + if (self.closed) { + return; + } + if ([anchor isKindOfClass:[ARPlaneAnchor class]]) { + [self firePlaneEvent:1 anchor:(ARPlaneAnchor *) anchor]; + } else { + [self fireAnchorUpdated:anchor]; + if ([anchor isKindOfClass:[ARFaceAnchor class]]) { + [self fireFaceRegions:(ARFaceAnchor *) anchor]; + } + } +} + +- (void)renderer:(id)renderer didRemoveNode:(SCNNode *)node + forAnchor:(ARAnchor *)anchor { + if (self.closed) { + return; + } + NSString *anchorId = anchor.identifier.UUIDString; + dispatch_async(dispatch_get_main_queue(), ^{ + [self.anchorNodes removeObjectForKey:anchorId]; + [self.contentNodes removeObjectForKey:anchorId]; + }); + if ([anchor isKindOfClass:[ARPlaneAnchor class]]) { + [self firePlaneEvent:2 anchor:(ARPlaneAnchor *) anchor]; + } else { + JAVA_OBJECT idj = fromNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + com_codename1_impl_ios_IOSARImpl_onAnchorRemoved___java_lang_String( + CN1_THREAD_GET_STATE_PASS_ARG idj); + } +} + +#pragma mark - ARSessionDelegate + +- (void)session:(ARSession *)session didUpdateFrame:(ARFrame *)frame { + if (self.closed) { + return; + } + // Throttle the per-frame pose + light delivery to roughly 10 Hz; the + // Java side only caches the latest value for polling. + self.frameCounter++; + if (self.frameCounter % 6 != 0) { + return; + } + float pose[7]; + cn1arPose(frame.camera.transform, pose); + float intensity = 1.0f; + float r = 1.0f; + float g = 1.0f; + float b = 1.0f; + JAVA_BOOLEAN lightValid = 0; + if (frame.lightEstimate != nil) { + // ARKit reports about 1000 lumens in neutral lighting; normalize so + // 1.0 is neutral (the ARLightEstimate convention). + intensity = (float) (frame.lightEstimate.ambientIntensity / 1000.0); + lightValid = 1; + } + com_codename1_impl_ios_IOSARImpl_onCameraFrame___float_float_float_float_float_float_float_float_float_float_float_boolean( + CN1_THREAD_GET_STATE_PASS_ARG + pose[0], pose[1], pose[2], pose[3], pose[4], pose[5], pose[6], + intensity, r, g, b, lightValid); +} + +- (void)session:(ARSession *)session cameraDidChangeTrackingState:(ARCamera *)camera { + if (self.closed) { + return; + } + int state; + int reason = CN1AR_REASON_NONE; + switch (camera.trackingState) { + case ARTrackingStateNotAvailable: + state = CN1AR_STATE_NOT_TRACKING; + break; + case ARTrackingStateLimited: + state = CN1AR_STATE_LIMITED; + switch (camera.trackingStateReason) { + case ARTrackingStateReasonInitializing: + case ARTrackingStateReasonRelocalizing: + reason = CN1AR_REASON_INITIALIZING; + break; + case ARTrackingStateReasonExcessiveMotion: + reason = CN1AR_REASON_EXCESSIVE_MOTION; + break; + case ARTrackingStateReasonInsufficientFeatures: + reason = CN1AR_REASON_INSUFFICIENT_FEATURES; + break; + default: + reason = CN1AR_REASON_INITIALIZING; + break; + } + break; + default: + state = CN1AR_STATE_TRACKING; + break; + } + com_codename1_impl_ios_IOSARImpl_onTrackingStateChanged___int_int( + CN1_THREAD_GET_STATE_PASS_ARG state, reason); +} + +@end + +#endif // INCLUDE_CN1_AR + +#pragma mark - IOSNative bridge functions +// Each `native ... cn1Ar*` declared on IOSNative.java has a matching C +// function below. Naming follows the standard ParparVM mangling: +// com_codename1_impl_ios_IOSNative____..._R_ +// Sessions are referenced by their CN1AR Objective-C pointer cast to +// JAVA_LONG; we retain in cn1ArCreate and release in cn1ArClose. When AR +// support is compiled out (INCLUDE_CN1_AR undefined, or tvOS/watchOS) the +// bridge symbols still exist so ParparVM links cleanly -- they just return +// null/0 and the Java side reports AR as unsupported. + +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH +#define CN1AR_PEER(sessionPeer) ((__bridge CN1AR *)(void *)(sessionPeer)) +#ifndef CN1_USE_ARC +#undef CN1AR_PEER +#define CN1AR_PEER(sessionPeer) ((CN1AR *)(sessionPeer)) +#endif +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1ArIsSupported___int_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT configType) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + return [CN1AR isConfigSupported:configType] ? 1 : 0; +#else + return 0; +#endif +} + +JAVA_LONG com_codename1_impl_ios_IOSNative_cn1ArCreate___R_long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + CN1AR *ar = [[CN1AR alloc] init]; +#ifndef CN1_USE_ARC + return (JAVA_LONG)[ar retain]; +#else + return (JAVA_LONG)(__bridge_retained void *)ar; +#endif +#else + return 0; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArAddReferenceImage___long_byte_1ARRAY_java_lang_String_float( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_OBJECT encodedImage, JAVA_OBJECT name, JAVA_FLOAT widthMeters) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + JAVA_ARRAY arr = (JAVA_ARRAY) encodedImage; + NSData *data = [NSData dataWithBytes:arr->data length:(NSUInteger) arr->length]; + NSString *nameStr = toNSString(CN1_THREAD_GET_STATE_PASS_ARG name); + [CN1AR_PEER(sessionPeer) addReferenceImage:data name:nameStr width:widthMeters]; +#endif +} + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1ArStart___long_int_int_boolean_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_INT configType, JAVA_INT planeMask, JAVA_BOOLEAN lightEstimation) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + return [CN1AR_PEER(sessionPeer) startWithConfigType:configType planeMask:planeMask + lightEstimation:lightEstimation] ? 1 : 0; +#else + return 0; +#endif +} + +JAVA_LONG com_codename1_impl_ios_IOSNative_cn1ArCreateView___long_R_long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + UIView *v = [CN1AR_PEER(sessionPeer) createView]; +#ifndef CN1_USE_ARC + return (JAVA_LONG)[v retain]; +#else + return (JAVA_LONG)(__bridge_retained void *)v; +#endif +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1ArHitTest___long_float_float_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_FLOAT xNorm, JAVA_FLOAT yNorm) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *s = [CN1AR_PEER(sessionPeer) hitTestX:xNorm y:yNorm]; + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG s); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1ArCreateAnchor___long_float_float_float_float_float_float_float_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_FLOAT tx, JAVA_FLOAT ty, JAVA_FLOAT tz, + JAVA_FLOAT qx, JAVA_FLOAT qy, JAVA_FLOAT qz, JAVA_FLOAT qw) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *s = [CN1AR_PEER(sessionPeer) createAnchorTx:tx ty:ty tz:tz + qx:qx qy:qy qz:qz qw:qw]; + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG s); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1ArCreateAnchorFromHit___long_int_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_INT hitId) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *s = [CN1AR_PEER(sessionPeer) createAnchorFromHit:hitId]; + if (s == nil) { + return JAVA_NULL; + } + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG s); +#else + return JAVA_NULL; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArRemoveAnchor___long_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_OBJECT anchorId) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *idStr = toNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + [CN1AR_PEER(sessionPeer) removeAnchorById:idStr]; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArClearAnchorContent___long_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_OBJECT anchorId) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *idStr = toNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + [CN1AR_PEER(sessionPeer) clearAnchorContent:idStr]; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArAddAnchorMesh___long_java_lang_String_float_1ARRAY_int_int_1ARRAY_int_int_byte_1ARRAY_float_1ARRAY( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer, + JAVA_OBJECT anchorId, JAVA_OBJECT interleaved, JAVA_INT vertexCount, + JAVA_OBJECT indices, JAVA_INT indexCount, JAVA_INT argbColor, + JAVA_OBJECT encodedTexture, JAVA_OBJECT localTransform16) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + NSString *idStr = toNSString(CN1_THREAD_GET_STATE_PASS_ARG anchorId); + JAVA_ARRAY vArr = (JAVA_ARRAY) interleaved; + JAVA_ARRAY iArr = (JAVA_ARRAY) indices; + JAVA_ARRAY tArr = (JAVA_ARRAY) localTransform16; + NSData *texture = nil; + if (encodedTexture != JAVA_NULL) { + JAVA_ARRAY texArr = (JAVA_ARRAY) encodedTexture; + texture = [NSData dataWithBytes:texArr->data length:(NSUInteger) texArr->length]; + } + [CN1AR_PEER(sessionPeer) addAnchorMesh:idStr + interleaved:(const float *) vArr->data + vertexCount:vertexCount + indices:(const int *) iArr->data + indexCount:indexCount + argb:argbColor + encodedTexture:texture + localTransform:(const float *) tArr->data]; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArPause___long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + [CN1AR_PEER(sessionPeer) pauseSession]; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArResume___long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + [CN1AR_PEER(sessionPeer) resumeSession]; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1ArClose___long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_LONG sessionPeer) { +#if defined(INCLUDE_CN1_AR) && !TARGET_OS_TV && !TARGET_OS_WATCH + if (sessionPeer == 0) { + return; + } + [CN1AR_PEER(sessionPeer) closeSession]; +#ifndef CN1_USE_ARC + [CN1AR_PEER(sessionPeer) release]; +#else + CN1AR *ar = (__bridge_transfer CN1AR *)(void *) sessionPeer; + ar = nil; +#endif +#endif +} diff --git a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h index 5d9d13dd1c9..06efb879e1d 100644 --- a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h +++ b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h @@ -110,6 +110,19 @@ void cn1RunSyncOnMainQueue(void (^block)(void)); #undef INCLUDE_CN1_CAMERA #endif +// INCLUDE_CN1_AR gates the com.codename1.ar native bridge (CN1AR.{h,m}: +// ARKit ARSession world/image/face tracking composited through an ARSCNView). +// IPhoneBuilder uncomments this only when the classpath scanner saw +// com.codename1.ar.*, so apps that never touch AR ship without any ARKit or +// SceneKit symbols and pass Apple's API-usage scan. +//#define INCLUDE_CN1_AR +// ARKit is unavailable on watchOS and tvOS; undo the define on those slices +// from this central header (included first by every AR TU) so the whole AR +// path compiles out consistently. +#if TARGET_OS_WATCH || TARGET_OS_TV +#undef INCLUDE_CN1_AR +#endif + // CN1_USE_CARPLAY gates the Apple CarPlay native bridge // (CodenameOne_CarPlaySceneDelegate.{h,m} + the IOSNative carPlay* trampolines: // CarPlay.framework, the CPTemplate translation). IPhoneBuilder uncomments this diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSARImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSARImpl.java new file mode 100644 index 00000000000..cd7b9a69cf4 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSARImpl.java @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.ios; + +import com.codename1.ar.ARCapabilities; +import com.codename1.ar.ARFaceRegion; +import com.codename1.ar.ARHitResult; +import com.codename1.ar.ARImageAnchor; +import com.codename1.ar.ARFaceAnchor; +import com.codename1.ar.ARLightEstimate; +import com.codename1.ar.ARModel; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARPose; +import com.codename1.ar.ARReferenceImage; +import com.codename1.ar.ARSessionOptions; +import com.codename1.ar.ARTrackingFailureReason; +import com.codename1.ar.ARTrackingMode; +import com.codename1.ar.ARTrackingState; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Quaternion; +import com.codename1.impl.ARImpl; +import com.codename1.ui.Display; +import com.codename1.ui.EncodedImage; +import com.codename1.ui.Image; +import com.codename1.ui.PeerComponent; +import com.codename1.util.AsyncResource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/// iOS implementation of `ARImpl`. Bridges to `CN1AR.{h,m}` which wraps an +/// ARKit `ARSession` composited through an `ARSCNView`; anchored content is +/// flattened from the portable `ARNode` tree into raw mesh buffers that the +/// native side turns into SceneKit geometry. +/// +/// @hidden +public class IOSARImpl extends ARImpl { + + // One Java instance is the active AR session at any moment (the core + // enforces a single open session); native callbacks arrive from the + // ARKit session/renderer queues without context, so we route them + // through this static. + private static volatile IOSARImpl ACTIVE; + + private long sessionPeer; + private EventSink sink; + private final Map planes = new HashMap(); + private final Map textureCache = new HashMap(); + + @Override + public ARCapabilities getCapabilities() { + boolean world = IOSImplementation.nativeInstance.cn1ArIsSupported(0); + boolean face = IOSImplementation.nativeInstance.cn1ArIsSupported(1); + return new ARCapabilities(world, world, world, face, world || face); + } + + @Override + public void setEventSink(EventSink sink) { + this.sink = sink; + } + + @Override + public void open(ARSessionOptions opts) throws IOException { + long peer = IOSImplementation.nativeInstance.cn1ArCreate(); + if (peer == 0) { + throw new IOException("AR is not supported on this device"); + } + ARReferenceImage[] images = opts.getReferenceImages(); + for (int i = 0; i < images.length; i++) { + IOSImplementation.nativeInstance.cn1ArAddReferenceImage(peer, + images[i].getEncodedImage(), images[i].getName(), + images[i].getPhysicalWidthMeters()); + } + int configType = opts.getTrackingMode() == ARTrackingMode.FACE ? 1 : 0; + int planeMask = 0; + if (opts.getPlaneDetection().includesHorizontal()) { + planeMask |= 1; + } + if (opts.getPlaneDetection().includesVertical()) { + planeMask |= 2; + } + if (!IOSImplementation.nativeInstance.cn1ArStart(peer, configType, planeMask, + opts.isLightEstimation())) { + IOSImplementation.nativeInstance.cn1ArClose(peer); + throw new IOException("The requested AR configuration is not supported on this device"); + } + this.sessionPeer = peer; + ACTIVE = this; + } + + @Override + public PeerComponent createViewPeer() { + if (sessionPeer == 0) { + return null; + } + long viewPeer = IOSImplementation.nativeInstance.cn1ArCreateView(sessionPeer); + if (viewPeer == 0) { + return null; + } + return IOSImplementation.instance.createNativePeer(new long[]{viewPeer}); + } + + @Override + public void hitTest(float xNorm, float yNorm, AsyncResource result) { + String packed = sessionPeer == 0 ? null + : IOSImplementation.nativeInstance.cn1ArHitTest(sessionPeer, xNorm, yNorm); + Display.getInstance().callSerially( + new CompleteOnEdt(result, parseHits(packed))); + } + + /// Resolves an AsyncResource on the EDT. A named static class since it + /// only needs the resource and value, not the enclosing impl. + private static final class CompleteOnEdt implements Runnable { + private final AsyncResource result; + private final T value; + + CompleteOnEdt(AsyncResource result, T value) { + this.result = result; + this.value = value; + } + + @Override + public void run() { + result.complete(value); + } + } + + // packed: "hitId|type|tx|ty|tz|qx|qy|qz|qw|distance|planeId;..." + private ARHitResult[] parseHits(String packed) { + if (packed == null || packed.length() == 0) { + return new ARHitResult[0]; + } + List entries = splitChar(packed, ';'); + List out = new ArrayList(entries.size()); + for (int i = 0; i < entries.size(); i++) { + List f = splitChar(entries.get(i), '|'); + if (f.size() < 11) { + continue; + } + try { + int hitId = Integer.parseInt(f.get(0)); + int type = Integer.parseInt(f.get(1)); + ARPose pose = new ARPose( + Float.parseFloat(f.get(2)), Float.parseFloat(f.get(3)), + Float.parseFloat(f.get(4)), Float.parseFloat(f.get(5)), + Float.parseFloat(f.get(6)), Float.parseFloat(f.get(7)), + Float.parseFloat(f.get(8))); + float distance = Float.parseFloat(f.get(9)); + ARPlane plane = f.get(10).length() == 0 ? null : planes.get(f.get(10)); + ARHitResult.Type t = type == 0 ? ARHitResult.Type.PLANE + : type == 1 ? ARHitResult.Type.ESTIMATED_PLANE + : ARHitResult.Type.FEATURE_POINT; + out.add(new ARHitResult(pose, distance, t, plane, Integer.valueOf(hitId))); + } catch (NumberFormatException err) { + // Skip a malformed entry rather than failing the whole test. + } + } + return out.toArray(new ARHitResult[out.size()]); + } + + private static List splitChar(String s, char sep) { + List out = new ArrayList(); + int start = 0; + int len = s.length(); + for (int i = 0; i < len; i++) { + if (s.charAt(i) == sep) { + out.add(s.substring(start, i)); + start = i + 1; + } + } + out.add(s.substring(start)); + return out; + } + + @Override + public String createAnchor(ARPose pose) { + return IOSImplementation.nativeInstance.cn1ArCreateAnchor(sessionPeer, + pose.getTx(), pose.getTy(), pose.getTz(), + pose.getQx(), pose.getQy(), pose.getQz(), pose.getQw()); + } + + @Override + public String createAnchorFromHit(Object nativeHandle, ARPose pose) { + if (nativeHandle instanceof Integer) { + String id = IOSImplementation.nativeInstance.cn1ArCreateAnchorFromHit( + sessionPeer, ((Integer) nativeHandle).intValue()); + if (id != null) { + return id; + } + } + return createAnchor(pose); + } + + @Override + public void removeAnchor(String anchorId) { + if (sessionPeer != 0) { + IOSImplementation.nativeInstance.cn1ArRemoveAnchor(sessionPeer, anchorId); + } + } + + @Override + public void setAnchorNode(String anchorId, ARNode node) { + if (sessionPeer == 0) { + return; + } + IOSImplementation.nativeInstance.cn1ArClearAnchorContent(sessionPeer, anchorId); + if (node != null) { + flatten(anchorId, node, Matrix4.identity()); + } + } + + @Override + public void nodeChanged(String anchorId, ARNode node) { + // Rebuild the whole subtree; placement-style content is small so the + // rebuild is cheap and keeps the bridge surface minimal. + setAnchorNode(anchorId, node); + } + + /// Walks the node tree accumulating local transforms and hands every + /// model mesh to the native renderer in anchor-local space. + private void flatten(String anchorId, ARNode node, float[] parentMatrix) { + if (!node.isVisible()) { + return; + } + float[] local = Matrix4.translation(node.getLocalX(), node.getLocalY(), node.getLocalZ()); + float[] rot = new float[16]; + Quaternion.toMatrix(new float[]{node.getLocalQx(), node.getLocalQy(), + node.getLocalQz(), node.getLocalQw()}, rot); + float[] tmp = new float[16]; + Matrix4.multiply(local, rot, tmp); + float s = node.getLocalScale(); + float[] m = new float[16]; + Matrix4.multiply(tmp, Matrix4.scaling(s, s, s), m); + float[] world = new float[16]; + Matrix4.multiply(parentMatrix, m, world); + + ARModel model = node.getModel(); + if (model != null) { + Mesh mesh = model.getMesh(); + if (mesh != null && mesh.getVertices().getFormat().getFloatsPerVertex() == 8) { + int vertexCount = mesh.getVertices().getVertexCount(); + float[] interleaved = mesh.getVertices().getData(); + int[] indices; + if (mesh.getIndices() != null) { + short[] shortIdx = mesh.getIndices().getData(); + indices = new int[mesh.getIndices().getIndexCount()]; + for (int i = 0; i < indices.length; i++) { + indices[i] = shortIdx[i] & 0xffff; + } + } else { + indices = new int[vertexCount]; + for (int i = 0; i < vertexCount; i++) { + indices[i] = i; + } + } + IOSImplementation.nativeInstance.cn1ArAddAnchorMesh(sessionPeer, anchorId, + interleaved, vertexCount, indices, indices.length, + model.getColor(), textureBytes(model), world); + } + } + for (int i = 0; i < node.getChildCount(); i++) { + flatten(anchorId, node.getChildAt(i), world); + } + } + + private byte[] textureBytes(ARModel model) { + if (textureCache.containsKey(model)) { + return textureCache.get(model); + } + byte[] bytes = null; + Image img = model.getBaseColorImage(); + if (img != null) { + if (img instanceof EncodedImage) { + bytes = ((EncodedImage) img).getImageData(); + } else { + bytes = EncodedImage.createFromImage(img, false).getImageData(); + } + } + textureCache.put(model, bytes); + return bytes; + } + + @Override + public void requestPermissions(AsyncResource result) { + // iOS prompts for the camera permission when the AR session first + // runs; there is no separate pre-flight, so report available. + Display.getInstance().callSerially(new CompleteOnEdt(result, Boolean.TRUE)); + } + + @Override + public void pause() { + if (sessionPeer != 0) { + IOSImplementation.nativeInstance.cn1ArPause(sessionPeer); + } + } + + @Override + public void resume() { + if (sessionPeer != 0) { + IOSImplementation.nativeInstance.cn1ArResume(sessionPeer); + } + } + + @Override + public void close() { + long s = sessionPeer; + sessionPeer = 0; + if (ACTIVE == this) { + ACTIVE = null; + } + planes.clear(); + textureCache.clear(); + if (s != 0) { + IOSImplementation.nativeInstance.cn1ArClose(s); + } + } + + // --------------------------------------------------------------------- + // Native -> Java callbacks. Called from CN1AR.m on the ARKit session / + // SceneKit renderer queues; the core session bridge marshals to the EDT. + // --------------------------------------------------------------------- + + private static ARTrackingState stateFor(int code) { + switch (code) { + case 0: + return ARTrackingState.NOT_TRACKING; + case 1: + return ARTrackingState.LIMITED; + default: + return ARTrackingState.TRACKING; + } + } + + private static ARTrackingFailureReason reasonFor(int code) { + switch (code) { + case 1: + return ARTrackingFailureReason.INITIALIZING; + case 2: + return ARTrackingFailureReason.EXCESSIVE_MOTION; + case 3: + return ARTrackingFailureReason.INSUFFICIENT_LIGHT; + case 4: + return ARTrackingFailureReason.INSUFFICIENT_FEATURES; + default: + return ARTrackingFailureReason.NONE; + } + } + + /// Called from native code when the camera tracking quality changes. + public static void onTrackingStateChanged(int state, int reason) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onTrackingStateChanged(stateFor(state), reasonFor(reason)); + } + + /// Called from native code on plane add (kind 0), update (1), remove (2). + public static void onPlaneEvent(int kind, String planeId, int planeType, + float tx, float ty, float tz, + float qx, float qy, float qz, float qw, + float extentX, float extentZ) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + if (kind == 2) { + self.planes.remove(planeId); + self.sink.onPlaneRemoved(planeId); + return; + } + ARPlane.Type type = planeType == 2 ? ARPlane.Type.VERTICAL + : planeType == 1 ? ARPlane.Type.HORIZONTAL_DOWN + : ARPlane.Type.HORIZONTAL_UP; + ARPlane plane = new ARPlane(planeId, type, + new ARPose(tx, ty, tz, qx, qy, qz, qw), + extentX, extentZ, null, ARTrackingState.TRACKING); + self.planes.put(planeId, plane); + if (kind == 0) { + self.sink.onPlaneAdded(plane); + } else { + self.sink.onPlaneUpdated(plane); + } + } + + /// Called from native code when an anchor's pose or tracking refines. + public static void onAnchorUpdated(String anchorId, + float tx, float ty, float tz, + float qx, float qy, float qz, float qw, + int state) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onAnchorUpdated(anchorId, + new ARPose(tx, ty, tz, qx, qy, qz, qw), stateFor(state)); + } + + /// Called from native code when the platform removes an anchor. + public static void onAnchorRemoved(String anchorId) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onAnchorRemoved(anchorId); + } + + /// Called from native code when a registered reference image is detected. + public static void onImageAnchorAdded(String anchorId, String imageName, + float tx, float ty, float tz, + float qx, float qy, float qz, float qw, + float physicalWidth) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onAnchorAdded(new ARImageAnchor(anchorId, + new ARPose(tx, ty, tz, qx, qy, qz, qw), imageName, physicalWidth)); + } + + /// Called from native code when a face is detected (FACE tracking mode). + public static void onFaceAnchorAdded(String anchorId, + float tx, float ty, float tz, + float qx, float qy, float qz, float qw) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onAnchorAdded(new ARFaceAnchor(anchorId, + new ARPose(tx, ty, tz, qx, qy, qz, qw))); + } + + /// Called from native code with world poses for the eye regions, packed as + /// "lx,ly,lz,lqx,lqy,lqz,lqw;rx,...". ARKit does not supply nose/forehead + /// region transforms so those stay null on iOS. + public static void onFaceRegionsUpdated(String anchorId, String packed) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null || packed == null) { + return; + } + List parts = splitChar(packed, ';'); + if (parts.size() < 2) { + return; + } + ARPose[] regions = new ARPose[ARFaceRegion.values().length]; + regions[ARFaceRegion.LEFT_EYE.ordinal()] = parsePose(parts.get(0)); + regions[ARFaceRegion.RIGHT_EYE.ordinal()] = parsePose(parts.get(1)); + self.sink.onFaceAnchorUpdated(anchorId, null, null, regions, null, null); + } + + private static ARPose parsePose(String csv) { + List f = splitChar(csv, ','); + if (f.size() < 7) { + return null; + } + try { + return new ARPose(Float.parseFloat(f.get(0)), Float.parseFloat(f.get(1)), + Float.parseFloat(f.get(2)), Float.parseFloat(f.get(3)), + Float.parseFloat(f.get(4)), Float.parseFloat(f.get(5)), + Float.parseFloat(f.get(6))); + } catch (NumberFormatException err) { + return null; + } + } + + /// Called from native code with the throttled per-frame camera pose and + /// light estimate. + public static void onCameraFrame(float tx, float ty, float tz, + float qx, float qy, float qz, float qw, + float lightIntensity, float lightR, + float lightG, float lightB, boolean lightValid) { + IOSARImpl self = ACTIVE; + if (self == null || self.sink == null) { + return; + } + self.sink.onCameraPose(new ARPose(tx, ty, tz, qx, qy, qz, qw)); + if (lightValid) { + self.sink.onLightEstimate(new ARLightEstimate(true, lightIntensity, + lightR, lightG, lightB)); + } + } +} diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index 0b903b45139..f66b94d808c 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -3992,6 +3992,15 @@ public com.codename1.impl.CameraImpl createCameraImpl() { return new IOSCameraImpl(); } + @Override + public com.codename1.impl.ARImpl createARImpl() { + if (!nativeInstance.cn1ArIsSupported(0) && !nativeInstance.cn1ArIsSupported(1)) { + // AR compiled out (non-AR app, tvOS/watchOS) or unsupported device. + return null; + } + return new IOSARImpl(); + } + @Override public String [] getAvailableRecordingMimeTypes() { // All of these amount to the same thing. diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index a4d0683d927..e395711085e 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -495,6 +495,27 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native void cn1CameraResume(long sessionPeer); native void cn1CameraClose(long sessionPeer); + // Augmented reality API (com.codename1.ar). Backed by CN1AR.m which wraps + // an ARKit ARSession composited through an ARSCNView. The IOSARImpl class + // on the Java side routes static callbacks delivered from the session and + // renderer queues. Sessions are referenced by the retained CN1AR pointer + // cast to long; configType is 0 for world tracking, 1 for face tracking; + // planeMask is bit 0 horizontal, bit 1 vertical. + native boolean cn1ArIsSupported(int configType); + native long cn1ArCreate(); + native void cn1ArAddReferenceImage(long sessionPeer, byte[] encodedImage, String name, float physicalWidthMeters); + native boolean cn1ArStart(long sessionPeer, int configType, int planeMask, boolean lightEstimation); + native long cn1ArCreateView(long sessionPeer); + native String cn1ArHitTest(long sessionPeer, float xNorm, float yNorm); + native String cn1ArCreateAnchor(long sessionPeer, float tx, float ty, float tz, float qx, float qy, float qz, float qw); + native String cn1ArCreateAnchorFromHit(long sessionPeer, int hitId); + native void cn1ArRemoveAnchor(long sessionPeer, String anchorId); + native void cn1ArClearAnchorContent(long sessionPeer, String anchorId); + native void cn1ArAddAnchorMesh(long sessionPeer, String anchorId, float[] interleaved, int vertexCount, int[] indices, int indexCount, int argbColor, byte[] encodedTexture, float[] localTransform16); + native void cn1ArPause(long sessionPeer); + native void cn1ArResume(long sessionPeer); + native void cn1ArClose(long sessionPeer); + // --------------------------------------------------------------------- // Portable 3D API (com.codename1.gpu) Metal backend. Backed by CN1GL3D.m. // Buffers are created over SIMD aligned Java arrays so Metal can wrap them diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java new file mode 100644 index 00000000000..155e51db685 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java @@ -0,0 +1,87 @@ +package com.codenameone.developerguide.ar; + +import com.codename1.ar.AR; +import com.codename1.ar.ARAnchor; +import com.codename1.ar.ARAnchorEvent; +import com.codename1.ar.ARImageAnchor; +import com.codename1.ar.ARModel; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARReferenceImage; +import com.codename1.ar.ARSession; +import com.codename1.ar.ARSessionOptions; +import com.codename1.ar.ARView; +import com.codename1.components.ToastBar; +import com.codename1.gpu.Primitives; +import com.codename1.ui.Form; +import com.codename1.ui.layouts.BorderLayout; + +/** + * Snippets that accompany the Augmented Reality guide chapter. Each method + * body between the tag markers is included verbatim into the AsciiDoc. + */ +public class ArSnippets { + + public void placeModelOnFloor(final byte[] modelBytes) { + // tag::placeModel[] + if (!AR.isSupported()) { + ToastBar.showInfoMessage("AR is not supported on this device"); + return; + } + final ARSession session = AR.open(new ARSessionOptions()); + final ARView view = session.createView(); + + Form f = new Form("AR", new BorderLayout()); + f.add(BorderLayout.CENTER, view); + f.show(); + + // modelBytes is a .glb asset, e.g. loaded from getResourceAsStream + view.addPointerReleasedListener(e -> { + float xn = (e.getX() - view.getAbsoluteX()) / (float) view.getWidth(); + float yn = (e.getY() - view.getAbsoluteY()) / (float) view.getHeight(); + session.hitTest(xn, yn).ready(hits -> { + if (hits.length > 0) { + ARAnchor anchor = hits[0].createAnchor(); + anchor.setNode(new ARNode(ARModel.fromGltf(modelBytes))); + } + }); + }); + // end::placeModel[] + } + + public void logPlanes(ARSession session) { + // tag::planeListener[] + session.addPlaneListener(ev -> { + ARPlane p = ev.getPlane(); + System.out.println(ev.getKind() + " " + p.getType() + + " " + p.getExtentX() + "x" + p.getExtentZ() + "m"); + }); + // end::planeListener[] + } + + public void anchorContent(ARAnchor anchor) { + // tag::anchorContent[] + ARNode root = new ARNode(ARModel.fromMesh( + Primitives.sphere(0.15f, 24, 32, false), 0xffdd4433)); + root.setLocalPosition(0, 0.15f, 0); // rest on the surface, not in it + anchor.setNode(root); + // end::anchorContent[] + } + + public void trackImage(final byte[] posterPngBytes, final byte[] overlayModel) { + // tag::imageTracking[] + ARSessionOptions opts = new ARSessionOptions().referenceImages(new ARReferenceImage[]{ + new ARReferenceImage("poster", posterPngBytes, 0.42f) // physical width in meters + }); + ARSession session = AR.open(opts); + session.addAnchorListener(ev -> { + if (ev.getKind() == ARAnchorEvent.Kind.ADDED + && ev.getAnchor() instanceof ARImageAnchor) { + ARImageAnchor img = (ARImageAnchor) ev.getAnchor(); + System.out.println("Recognized " + img.getReferenceImageName()); + img.setNode(new ARNode(ARModel.fromGltf(overlayModel))); + } + }); + // end::imageTracking[] + } +} diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java new file mode 100644 index 00000000000..e97f4537660 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java @@ -0,0 +1,72 @@ +package com.codenameone.developerguide.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Material; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Primitives; +import com.codename1.gpu.Texture; +import com.codename1.ui.EncodedImage; +import com.codename1.ui.Form; +import com.codename1.ui.layouts.BorderLayout; +import com.codename1.vr.Media360View; +import com.codename1.vr.TextureSource; +import com.codename1.vr.VREye; +import com.codename1.vr.VRRenderer; +import com.codename1.vr.VRView; +import java.io.IOException; + +/** + * Snippets that accompany the Virtual Reality and 360 Media guide chapter. + */ +public class VrSnippets { + + public void showStereoScene() { + // tag::stereoScene[] + VRView vr = new VRView(new VRRenderer() { + Mesh cube; + Material material; + + public void onInit(GraphicsDevice device) { + cube = Primitives.cube(device, 0.5f); + material = new Material(Material.Type.PHONG).setColor(0xff3366ff); + } + + public void onEyeFrame(GraphicsDevice device, VREye eye, Camera camera) { + device.draw(cube, material, Matrix4.translation(0, 0, -2)); + } + + public void onDispose(GraphicsDevice device) { } + }); + vr.setContinuous(true); // head tracked scenes want continuous rendering + + Form f = new Form("VR", new BorderLayout()); + f.add(BorderLayout.CENTER, vr); + f.show(); + // end::stereoScene[] + } + + public void showPanorama(Form f) throws IOException { + // tag::photoViewer[] + Media360View pano = new Media360View(); + pano.setImage(EncodedImage.create("/panorama.jpg")); + f.add(BorderLayout.CENTER, pano); + // end::photoViewer[] + } + + public void dynamicTexture(Media360View pano, final int[] initialPixels) { + // tag::textureSource[] + pano.setTextureSource(new TextureSource() { + public Texture createTexture(GraphicsDevice device) { + return device.createTexture(1024, 512, initialPixels); + } + public boolean updateTexture(GraphicsDevice device, Texture texture) { + // mutate pixels and re-upload as needed; return true when changed + return false; + } + public void dispose(GraphicsDevice device) { } + }); + // end::textureSource[] + } +} diff --git a/docs/developer-guide/Augmented-Reality.asciidoc b/docs/developer-guide/Augmented-Reality.asciidoc new file mode 100644 index 00000000000..71d1d5c15f9 --- /dev/null +++ b/docs/developer-guide/Augmented-Reality.asciidoc @@ -0,0 +1,123 @@ +== Augmented Reality + +The `com.codename1.ar` package composites virtual 3D content into the live camera image: the session tracks the device's position in the real world, detects surfaces, recognizes registered images and faces, and keeps virtual objects locked to real-world positions as the user moves. It's backed by ARKit on iOS and by ARCore (Google Play Services for AR) on Android, behind one portable API. + +Referencing any class in the package is enough to wire the build: the class scanner injects the camera permission, the `NSCameraUsageDescription` plist entry (override the text with the `ios.NSCameraUsageDescription` build hint), links ARKit and SceneKit on iOS and adds the ARCore dependency plus the `com.google.ar.core` manifest entry on Android. Apps that never touch the package pay no size, permission or store-visibility cost. By default the Android manifest marks AR `optional` so the app still installs on devices without ARCore; set the `android.ar.required=true` build hint to make the app AR-only. + +[WARNING] +==== +Always guard AR functionality behind `AR.isSupported()`. Older devices, tvOS, watchOS, desktop builds and browsers report `false`; feature-specific hardware such as face tracking is gated through `AR.getCapabilities()`. +==== + +=== Concepts + +[cols="1,3"] +|=== +|Type |Role + +|`AR` +|Static entry point: support and capability checks, permission request, session opening. + +|`ARSession` +|The running AR experience. One session may be open at a time; it owns the view, the anchors and the detected planes, and delivers all events. + +|`ARView` +|The component that renders the camera image composited with the anchored content. Add it to a form like any other component. + +|`ARPose` +|An immutable position + rotation in world space: meters, right-handed, Y up, -Z forward from the initial camera direction. Converts to a `com.codename1.gpu.Matrix4` compatible matrix. + +|`ARPlane` +|A detected real-world surface (floor, table, wall) that grows and refines as the session learns the environment. + +|`ARHitResult` +|The intersection of a screen-point ray with real-world geometry; the bridge from a tap to a world position. + +|`ARAnchor` +|A tracked fixed point in the real world. The attachment point for content; also delivered for recognized images (`ARImageAnchor`) and faces (`ARFaceAnchor`). + +|`ARNode` / `ARModel` +|The content placed at an anchor: a small scene graph of nodes carrying glTF models or `com.codename1.gpu.Mesh` geometry. + +|`ARLightEstimate` +|The real-world lighting estimate, polled per frame to shade content so it blends in. +|=== + +=== Getting started: Place a model on the floor + +The canonical AR loop is: open a session, show its view, wait for a plane, hit test a tap, anchor content at the hit. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java[tag=placeModel,indent=0] +---- + +`ARSessionOptions` configures the session before it opens: the tracking mode (`WORLD` or `FACE`), which plane orientations to detect, whether to estimate lighting, and the reference images to recognize. The defaults open a world tracking session with horizontal plane detection and light estimation, which suits the place-content-on-a-surface use case above. + +=== Plane detection + +Planes are delivered through `ARPlaneListener` as immutable snapshots: an `ADDED` event when a new surface is found, `UPDATED` snapshots (sharing the same `getId()`) as the surface grows, and `REMOVED` when a plane merges into another. `ARSession.getPlanes()` always reflects the latest snapshots. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java[tag=planeListener,indent=0] +---- + +The plane's local frame has X/Z spanning the surface and Y along the normal; `getCenterPose()` positions that frame in the world. `ARPlane.Type` distinguishes upward-facing horizontal surfaces (floors, tables), downward-facing ones (ceilings) and vertical surfaces (walls) - request vertical detection with `ARSessionOptions.planeDetection(ARPlaneDetection.HORIZONTAL_AND_VERTICAL)`. + +=== Hit testing and anchors + +`hitTest(xNorm, yNorm)` shoots a ray from a normalized view coordinate into the world and resolves - on the EDT - with the intersections ordered nearest first. Prefer `ARHitResult.createAnchor()` over `session.createAnchor(pose)` when placing on detected geometry: the platform can then anchor to the exact native raycast, which tracks better as the environment refines. + +Anchors survive tracking refinements: the session updates `getPose()` over time and reports the change through `ARAnchorListener`. Remove an anchor (and its content) with `detach()`. Content attaches through `setNode(ARNode)`: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java[tag=anchorContent,indent=0] +---- + +Node transforms are relative to the anchor, in meters. Nodes may nest (`addChild`), carry glTF assets (`ARModel.fromGltf`) or portable meshes (`ARModel.fromMesh`), and any mutation after attachment - position, rotation, scale, visibility, children - re-syncs the platform renderer automatically. + +=== Light estimation + +`session.getLightEstimate()` returns the latest estimate of the real-world lighting. It refreshes every frame without firing events, so poll it when rendering-relevant decisions are made. The ambient intensity is normalized so `1.0` means neutral indoor lighting; the color correction is a per-channel scale where `{1, 1, 1}` is neutral. Platform renderers already apply the estimate to anchored content; the API exposes it for custom logic such as prompting the user when the room is too dark. + +=== Image tracking + +Register printed images - posters, game boards, product labels - and the session anchors content to them when the camera sees them: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/ar/ArSnippets.java[tag=imageTracking,indent=0] +---- + +The physical width matters: the platform uses it to estimate the image's distance, so measure the real print. The anchor pose is centered on the physical image with local X/Z spanning its surface. + +=== Face tracking + +Face tracking is a distinct session configuration using the front camera; open it with `ARSessionOptions.trackingMode(ARTrackingMode.FACE)` after checking `AR.getCapabilities().isFaceTrackingSupported()`. Detected faces arrive as `ARFaceAnchor`s through the anchor listener, carrying the head pose plus optional finer geometry: + +- `getRegionPose(ARFaceRegion)` returns the pose of a named region. Availability differs per platform (ARCore supplies the nose tip and forehead regions, ARKit supplies the eyes), which is why code must always handle a `null` region. +- `getMeshVertices()` / `getMeshTriangles()` expose the tracked face mesh where the platform provides one, or `null` where it doesn't. + +=== Events and threading + +Every listener fires on the EDT and every getter reflects the state as of the events delivered so far, so AR code follows normal Codename One threading rules with no extra synchronization. High-frequency refinements (anchor and plane updates, camera pose, light estimate) are coalesced: a slow EDT sees the latest values rather than a backlog. This contrasts with the `com.codename1.gpu.Renderer` callbacks used by the VR API, which run on the render thread. + +=== The simulator and debugging + +The Codename One simulator ships a simulated AR backend modeled on the Android emulator's virtual scene, so the full AR loop is debuggable with breakpoints without a device. When a session opens, the simulator "detects" the floor (and, when requested, a wall) of a virtual room after a short startup delay, mimicking real AR initialization. The AR view renders the room and every anchored model; drive the virtual device camera with the mouse (drag to look around) and keyboard (`W`/`A`/`S`/`D` or the arrow keys to move, `Q`/`E` for up and down). + +image::img/ar-simulator-model.png[A model placed on the virtual room floor in the simulator,scaledwidth=30%] + +The Simulate menu's *AR Simulation* window drives the paths that are hard to reproduce on demand with real hardware: force degraded tracking states and failure reasons, change the light estimate, re-run plane detection, trigger recognition of a registered reference image, and toggle a simulated face anchor for `FACE` sessions. + +On devices, use the standard native tools: the generated Xcode project debugs the ARKit session and SceneKit scene directly (including Instruments), and Android Studio attaches to the Gradle project for ARCore logging. Note that Apple's iOS Simulator has no ARKit support - real ARKit behavior needs a physical device - while the Android emulator can run ARCore against its own virtual scene. + +=== Platform notes + +- *Dependencies*: injected automatically as described above. The ARCore dependency adds about 2 MB to Android binaries; iOS links the system ARKit/SceneKit frameworks at no size cost. +- *Capabilities*: face tracking requires a capable front camera (a TrueDepth camera on iOS). Check `ARCapabilities` per feature rather than assuming. +- *Face regions*: ARCore supplies `NOSE_TIP`, `FOREHEAD_LEFT` and `FOREHEAD_RIGHT`; ARKit supplies `LEFT_EYE` and `RIGHT_EYE`. Code that uses regions must handle a `null` region. +- *ARCore installation*: on Android, Play Services for AR may need installing or updating; when `AR.open` triggers that flow it throws with a message saying to retry once installation completes. +- *tvOS / watchOS*: AR compiles out entirely; `AR.isSupported()` reports `false`. diff --git a/docs/developer-guide/Virtual-Reality.asciidoc b/docs/developer-guide/Virtual-Reality.asciidoc new file mode 100644 index 00000000000..72cfea70f05 --- /dev/null +++ b/docs/developer-guide/Virtual-Reality.asciidoc @@ -0,0 +1,93 @@ +== Virtual Reality and 360 Media + +The `com.codename1.vr` package renders application scenes in stereoscopic VR and displays 360-degree panoramas, built entirely on the portable `com.codename1.gpu` pipeline and the `com.codename1.sensors` motion sensors. There is no platform SDK dependency: everything here runs wherever `Display.getInstance().isGpuSupported()` is true, which includes the simulator, and adds nothing to the build for apps that don't use it. + +[IMPORTANT] +==== +Like `com.codename1.gpu.Renderer`, the `VRRenderer` and `TextureSource` callbacks run on the platform render thread that owns the GPU context - never touch Codename One UI components from them. Everything else in the package is ordinary EDT-friendly component API. +==== + +=== Concepts + +[cols="1,3"] +|=== +|Type |Role + +|`VRView` +|The stereo component: clears the frame, splits the viewport per eye, positions the cameras from head tracking and invokes your renderer once per eye. + +|`VRRenderer` +|Your scene callback: `onInit`, `onEyeFrame(device, eye, camera)` and `onDispose`. + +|`VREye` +|Which eye a frame is for: `LEFT`, `RIGHT`, or `CENTER` in mono mode. + +|`VRSettings` +|Interpupillary distance (default 0.064m), per-eye field of view, clip planes and the stereo default. + +|`HeadTracker` +|Wires the gyroscope, accelerometer and magnetometer into an orientation with thread-safe snapshots for the render thread. + +|`OrientationFilter` +|The deterministic sensor-fusion math behind the tracker, usable on its own. + +|`VRCameraRig` +|Per-eye camera math over the existing `com.codename1.gpu.Camera`, usable without `VRView` for custom pipelines. + +|`Media360View` +|A 360-degree panorama viewer: an equirectangular image on the inside of a sphere with drag or gyroscope look-around. + +|`TextureSource` +|Extension point feeding dynamic textures (procedural animation, future video) to `Media360View`. +|=== + +=== Getting started: A stereo scene + +A `VRRenderer` draws the same scene for every eye; the camera differences produce the depth effect. The view configures the viewport and camera before each `onEyeFrame`, so the renderer only issues draw calls: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java[tag=stereoScene,indent=0] +---- + +`setStereo(false)` switches to a single centered viewpoint (`VREye.CENTER`) at runtime, useful for a non-headset preview of the same scene. `setPosition(x, y, z)` moves the viewer through the world; `setClearColor(argb)` sets the background. + +image::img/vr-stereo-scene.png[A stereo scene on an Android phone; the offset between the eye views is the depth cue,scaledwidth=30%] + +=== Head tracking + +`VRView` starts its `HeadTracker` when the component appears and stops it when it leaves the screen; the sensors draw power only between the two. The tracker fuses the gyroscope with the accelerometer (tilt reference) and, when present, the magnetometer (yaw reference) through a complementary filter - tune the blend via `getHeadTracker().getFilter().setGyroWeight(float)`. + +Call `recenter()` to make the current view direction the new "straight ahead," keeping pitch and roll - the standard VR recentering gesture. On devices without a gyroscope the tracker falls back to accelerometer tilt: pitch and roll stay accurate but yaw has no stable reference. `HeadTracker.isSupported()` reports whether any usable motion hardware exists; without it the orientation simply stays put and the scene still renders. + +=== The 360 photo viewer + +`Media360View` displays equirectangular panoramas - the format produced by 360 cameras and phone panorama modes, where longitude maps to X and latitude to Y: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java[tag=photoViewer,indent=0] +---- + +image::img/vr-360-panorama.png[An equirectangular panorama in side-by-side stereo,scaledwidth=40%] + +Dragging looks around with the grab-the-world gesture; `setHeadTrackingEnabled(true)` adds gyroscope look composed with the drag, and `setStereo(true)` renders side by side for cardboard-style viewers. Photo spheres are captured from a single point, so stereo intentionally uses zero eye separation - there is no parallax information in the image to reproduce. `setYaw`/`setPitch` position the view programmatically and `reset()` returns to straight ahead. + +=== 360 video and dynamic content + +There is no platform path from media frames to GPU textures yet, so 360 video isn't supported directly. `TextureSource` is the extension point for dynamic content: the view calls `createTexture` once on the render thread and `updateTexture` before every frame, so procedurally animated panoramas work today and a future media-to-texture bridge can plug in without API changes. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/vr/VrSnippets.java[tag=textureSource,indent=0] +---- + +=== The simulator + +Both `VRView` and `Media360View` run in the simulator through the desktop 3D backend, and the Simulate menu's *Motion / Gesture Simulation* window feeds synthetic sensor data to the `HeadTracker`, so head-tracked scenes are drivable from the desktop. Drag-look in `Media360View` works with the mouse as it does with a finger. + +=== Limitations and notes + +- *Lens distortion*: the stereo output is undistorted side-by-side rendering. Barrel distortion for lens-based viewers requires an offscreen render pass the portable GPU API doesn't expose yet, so there is no distortion setting to configure. +- *Battery*: continuous rendering plus live sensors is the most power-hungry mode a phone UI can run; stop or pause the view whenever it isn't visible (the component does this automatically on hide). +- *Coordinates*: the world frame matches the AR and 3D graphics APIs (meters, right-handed, Y up, -Z forward), letting content and math move between the three without conversion. diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index b1f02ccdffd..97a49572788 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -69,6 +69,10 @@ include::graphics.asciidoc[] include::3D-Graphics.asciidoc[] +include::Augmented-Reality.asciidoc[] + +include::Virtual-Reality.asciidoc[] + include::Game-Development.asciidoc[] include::Game-Builder.asciidoc[] diff --git a/docs/developer-guide/img/ar-simulator-model.png b/docs/developer-guide/img/ar-simulator-model.png new file mode 100644 index 00000000000..681da01a674 Binary files /dev/null and b/docs/developer-guide/img/ar-simulator-model.png differ diff --git a/docs/developer-guide/img/vr-360-panorama.png b/docs/developer-guide/img/vr-360-panorama.png new file mode 100644 index 00000000000..baa1cfc9928 Binary files /dev/null and b/docs/developer-guide/img/vr-360-panorama.png differ diff --git a/docs/developer-guide/img/vr-stereo-scene.png b/docs/developer-guide/img/vr-stereo-scene.png new file mode 100644 index 00000000000..4f628fe49b4 Binary files /dev/null and b/docs/developer-guide/img/vr-stereo-scene.png differ diff --git a/docs/developer-guide/languagetool-accept.txt b/docs/developer-guide/languagetool-accept.txt index f060459bb4d..8515409841c 100644 --- a/docs/developer-guide/languagetool-accept.txt +++ b/docs/developer-guide/languagetool-accept.txt @@ -669,3 +669,7 @@ foldables? bitmask DeX Siri + +# AR / VR terms (Augmented-Reality.asciidoc, Virtual-Reality.asciidoc) +raycasts? +equirectangular diff --git a/maven/android/pom.xml b/maven/android/pom.xml index a85d8f2b50a..2afe8497b10 100644 --- a/maven/android/pom.xml +++ b/maven/android/pom.xml @@ -74,6 +74,24 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + com/codename1/impl/android/ar/** + + + maven-antrun-plugin diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java index 7195b3ea47e..9f0bb2d9158 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java @@ -191,6 +191,24 @@ public final class AiDependencyTable { .markBigUpload() .description("On-device Stable Diffusion (local-build only)")); + // Cross-platform augmented reality (com.codename1.ar): ARKit + + // SceneKit on iOS (linked explicitly by IPhoneBuilder, gated by + // the INCLUDE_CN1_AR define since neither is default-linked), + // Google Play Services for AR (ARCore) on Android. The camera + // permission and plist string ride along because AR always + // drives the camera. The com.google.ar.core meta-data marks + // ARCore optional so the app still installs on non-AR devices; + // the android.ar.required=true build hint flips it to required. + e.add(new Entry("com/codename1/ar/") + .iosFrameworks("ARKit", "SceneKit") + .iosPlist("NSCameraUsageDescription", + "Used to display augmented reality content.") + .androidGradle("com.google.ar:core:1.44.0") + .androidPermissions("android.permission.CAMERA") + .androidFeatures("android.hardware.camera.ar") + .androidMetaData("com.google.ar.core", "optional") + .description("Cross-platform augmented reality (world/image/face tracking)")); + ENTRIES = Collections.unmodifiableList(e); } @@ -262,6 +280,7 @@ public static final class Entry { private final List androidGradle = new ArrayList(); private final List androidPermissions = new ArrayList(); private final List androidFeatures = new ArrayList(); + private final List androidMetaData = new ArrayList(); private boolean requiresBigUpload; private String description = ""; @@ -318,6 +337,11 @@ Entry androidFeatures(String... feats) { return this; } + Entry androidMetaData(String name, String value) { + androidMetaData.add(new String[]{name, value}); + return this; + } + Entry markBigUpload() { this.requiresBigUpload = true; return this; @@ -363,6 +387,13 @@ public List androidFeatures() { return Collections.unmodifiableList(androidFeatures); } + /** Each entry is {name, value}: an application-level manifest + * <meta-data> element the Android builder injects unless the + * app already declares the same name. */ + public List androidMetaDataEntries() { + return Collections.unmodifiableList(androidMetaData); + } + public boolean requiresBigUpload() { return requiresBigUpload; } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index cdf72b894ca..c945eba73c4 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -436,6 +436,7 @@ public File getGradleProjectDirectory() { private boolean useAndroidX; private boolean migrateToAndroidX; private boolean shouldIncludeGoogleImpl; + private boolean arSupport; static { isMac = System.getProperty("os.name").toLowerCase().contains("mac"); @@ -1304,6 +1305,12 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc @Override public void usesClass(String cls) { aiAcc.consume(cls); + if (cls.indexOf("com/codename1/ar/") == 0) { + // Keeps the ARCore-backed impl sources (deleted for + // non-AR apps below) and bumps minSdk to the ARCore + // floor. + arSupport = true; + } if (cls.indexOf("com/codename1/notifications") == 0) { recieveBootCompletedPermission = true; if (targetSDKVersionInt >= 33) { @@ -1560,17 +1567,40 @@ public void usesClassMethod(String cls, String method) { // aiExtraGradleDependencies and appended just before // additionalDependencies is written to build.gradle below. StringBuilder aiExtraGradleDependencies = new StringBuilder(); + String aiApplicationMetaData = ""; for (AiDependencyTable.Entry entry : aiAcc.hits()) { for (String perm : entry.androidPermissions()) { String addString = " \n"; xPermissions += permissionAdd(request, perm, addString); } for (String feat : entry.androidFeatures()) { - String addString = " \n"; + String required = "false"; + if ("android.hardware.camera.ar".equals(feat) + && "true".equals(request.getArg("android.ar.required", "false"))) { + // The app opts into being AR-only: the store then hides it + // from devices without ARCore support. + required = "true"; + } + String addString = " \n"; if (!xPermissions.contains("\n"; + } + } for (String gav : entry.androidGradleDeps()) { aiExtraGradleDependencies.append(" implementation '").append(gav).append("'\n"); } @@ -1578,6 +1608,10 @@ public void usesClassMethod(String cls, String method) { if (aiAcc.anyRequiresBigUpload()) { request.putArgument("cn1.ai.requiresBigUpload", "true"); } + if (arSupport) { + // ARCore requires API 24. + minSDK = maxInt("24", minSDK); + } // Inject USE_BIOMETRIC / USE_FINGERPRINT only when the app actually // touches com.codename1.security (Biometrics / SecureStorage). Both @@ -2129,6 +2163,22 @@ public void usesClassMethod(String cls, String method) { fb.delete(); } + if (!arSupport) { + // The ARCore-backed impl package compiles against com.google.ar + // classes that only exist when the AR gradle dependency is added, + // so it must be removed for apps that never touch + // com.codename1.ar (AndroidImplementation reaches it through + // reflection only). + File arPackage = new File(srcDir, "com/codename1/impl/android/ar"); + File[] arFiles = arPackage.listFiles(); + if (arFiles != null) { + for (File f : arFiles) { + f.delete(); + } + } + arPackage.delete(); + } + final String moPubAdUnitId = request.getArg("android.mopubId", null); if (moPubAdUnitId != null && moPubAdUnitId.length() > 0) { integrateMoPub = true; @@ -3113,6 +3163,7 @@ public void usesClassMethod(String cls, String method) { + pushManifestEntries + billingServiceData + wearApplicationMetaData + + aiApplicationMetaData + " " + request.getArg("android.xapplication", "") + mopubActivities + alarmRecevier diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index a28ddffa30e..87bca8c7a0a 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -114,6 +114,7 @@ public class IPhoneBuilder extends Executor { private boolean usesBiometrics; private boolean usesNfc; private boolean usesCn1Camera; + private boolean usesCn1Ar; // Set when the app references com.codename1.car.* (Apple CarPlay support). Gates the // CN1_USE_CARPLAY native define, CarPlay.framework linkage, the carplay entitlement and the // CarPlay scene in the Info.plist scene manifest. Apps that never touch the API see no change. @@ -829,6 +830,12 @@ public void usesClass(String cls) { if (!usesCn1Camera && cls.indexOf("com/codename1/camera/") == 0) { usesCn1Camera = true; } + // Augmented reality (com.codename1.ar.*). Gated on actual + // usage so ARKit/SceneKit and the CN1AR natives are only + // built for apps that reference the AR API. + if (!usesCn1Ar && cls.indexOf("com/codename1/ar/") == 0) { + usesCn1Ar = true; + } // Apple CarPlay (com.codename1.car.*). Gated on actual usage so the // CarPlay scene/entitlement/framework are only added for apps that // build an in-car experience. @@ -2289,6 +2296,31 @@ public void usesClassMethod(String cls, String method) { } } + // Augmented reality: uncomment INCLUDE_CN1_AR so the CN1AR + // natives (ARKit + ARSCNView) compile in, and link ARKit / + // SceneKit explicitly -- neither is default-linked and the + // AiDependencyTable iosFrameworks field is documentation-only. + // Apps that never reference com.codename1.ar leave the define + // commented out so no ARKit symbol is referenced, which keeps + // Apple's API-usage scan quiet and tvOS/watchOS slices clean. + if (usesCn1Ar) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_AR", + "#define INCLUDE_CN1_AR"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_AR", ex); + } + String arLibs = "ARKit.framework;SceneKit.framework"; + if (addLibs == null || addLibs.length() == 0) { + addLibs = arLibs; + } else if (!addLibs.toLowerCase().contains("arkit.framework")) { + addLibs = addLibs + ";" + arLibs; + } + } + // CarPlay: link CarPlay.framework (+ MediaPlayer for the now-playing template) and // inject the per-category carplay entitlement. The CarPlay entitlements are granted by // Apple per app category, so we only inject the ones the project opts into via the diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java index 8fbd604fae7..172b574222d 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java @@ -257,7 +257,10 @@ void signAndNotarizeMacApp(BuildRequest request, File appBundle, String parparvmOptionalFrameworksArg() { return "-Doptional.frameworks=AddressBookUI.framework;" + "AddressBook.framework;MessageUI.framework;" - + "MediaPlayer.framework;GLKit.framework;OpenGLES.framework"; + + "MediaPlayer.framework;GLKit.framework;OpenGLES.framework;" + // ARKit world tracking is unavailable on the Mac slice; linked + // on iOS when the app references com.codename1.ar. + + "ARKit.framework"; } String getIosMinDeploymentTarget() { diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/TvNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/TvNativeBuilder.java index e1df3c15a3c..c4077d31985 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/TvNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/TvNativeBuilder.java @@ -87,7 +87,10 @@ class TvNativeBuilder { "WebKit.framework;OpenGLES.framework;GLKit.framework;" // CarPlay.framework is iOS-only (absent on tvOS); it is linked on the iOS slice when the // app references com.codename1.car, so weak-link it for the tvOS slice. - + "CarPlay.framework"; + + "CarPlay.framework;" + // ARKit is iOS-only; it is linked on the iOS slice when the app references + // com.codename1.ar, so weak-link it for the tvOS slice. + + "ARKit.framework"; TvNativeBuilder(IPhoneBuilder owner) { this.owner = owner; diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java index 22b9a1b5620..c7a6adb888a 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/WatchNativeBuilder.java @@ -110,7 +110,10 @@ class WatchNativeBuilder { + "WebKit.framework;StoreKit.framework;" // CarPlay.framework is iOS-only (absent on watchOS); it is linked on the iOS slice when // the app references com.codename1.car, so weak-link it for the watch slice. - + "CarPlay.framework"; + + "CarPlay.framework;" + // ARKit and SceneKit are absent on watchOS; they are linked on the iOS slice when the + // app references com.codename1.ar, so weak-link them for the watch slice. + + "ARKit.framework;SceneKit.framework"; WatchNativeBuilder(IPhoneBuilder owner) { this.owner = owner; diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java index 5c366bef898..7aca82af21f 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java @@ -190,6 +190,48 @@ void accumulatorDeduplicates() { assertEquals(1, acc.hits().size()); } + @Test + void arApiInjectsArKitCameraAndArCore() { + List hits = AiDependencyTable.matchesFor( + "com/codename1/ar/AR"); + assertEquals(1, hits.size(), "expected the AR entry to fire"); + AiDependencyTable.Entry e = hits.get(0); + // iOS: ARKit + SceneKit (linked explicitly by IPhoneBuilder) and the + // camera usage string, overridable via ios.NSCameraUsageDescription. + assertTrue(e.iosFrameworks().contains("ARKit")); + assertTrue(e.iosFrameworks().contains("SceneKit")); + assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); + // Android: the ARCore dependency, the camera permission and the + // optional AR feature/meta-data pair so non-AR devices still install. + assertEquals(1, e.androidGradleDeps().size()); + assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.ar:core")); + assertTrue(e.androidPermissions().contains("android.permission.CAMERA")); + assertTrue(e.androidFeatures().contains("android.hardware.camera.ar")); + assertEquals(1, e.androidMetaDataEntries().size()); + assertEquals("com.google.ar.core", e.androidMetaDataEntries().get(0)[0]); + assertEquals("optional", e.androidMetaDataEntries().get(0)[1]); + } + + @Test + void arEntryMatchesTheWholePackageButNothingElse() { + assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARSession").size()); + assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARNode").size()); + // The pure-core VR package must NOT pull the AR native dependencies. + assertTrue(AiDependencyTable.matchesFor("com/codename1/vr/VRView").isEmpty()); + } + + @Test + void nonArEntriesCarryNoMetaData() { + // The meta-data field is new; make sure the existing entries did not + // accidentally gain one. + for (AiDependencyTable.Entry e : AiDependencyTable.entries()) { + if (!e.classPrefix().startsWith("com/codename1/ar/")) { + assertTrue(e.androidMetaDataEntries().isEmpty(), + e.classPrefix() + " should carry no manifest meta-data"); + } + } + } + private static String findPlistDefault(AiDependencyTable.Entry e, String key) { for (String[] entry : e.iosPlistEntries()) { if (key.equals(entry[0])) { diff --git a/maven/core-unittests/src/test/java/com/codename1/ar/ARModelTest.java b/maven/core-unittests/src/test/java/com/codename1/ar/ARModelTest.java new file mode 100644 index 00000000000..62149f77004 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ar/ARModelTest.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Primitives; +import com.codename1.junit.UITestBase; +import com.codename1.util.Base64; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for {@link ARModel}: glTF byte retention with lazy device-free + * parsing, and the mesh/color/texture factories. + */ +class ARModelTest extends UITestBase { + + // A single-triangle glTF built the same way as GltfLoaderTest. + + private static void putFloatLE(byte[] b, int p, float f) { + int bits = Float.floatToIntBits(f); + b[p] = (byte) (bits & 0xff); + b[p + 1] = (byte) ((bits >> 8) & 0xff); + b[p + 2] = (byte) ((bits >> 16) & 0xff); + b[p + 3] = (byte) ((bits >> 24) & 0xff); + } + + private static void putUShortLE(byte[] b, int p, int v) { + b[p] = (byte) (v & 0xff); + b[p + 1] = (byte) ((v >> 8) & 0xff); + } + + private static byte[] triangleGltf() { + byte[] buf = new byte[42]; + float[] pos = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + for (int i = 0; i < 9; i++) { + putFloatLE(buf, i * 4, pos[i]); + } + putUShortLE(buf, 36, 0); + putUShortLE(buf, 38, 1); + putUShortLE(buf, 40, 2); + String dataUri = "data:application/octet-stream;base64," + Base64.encodeNoNewline(buf); + String json = "{" + + "\"asset\":{\"version\":\"2.0\"}," + + "\"buffers\":[{\"byteLength\":42,\"uri\":\"" + dataUri + "\"}]," + + "\"bufferViews\":[" + + "{\"buffer\":0,\"byteOffset\":0,\"byteLength\":36}," + + "{\"buffer\":0,\"byteOffset\":36,\"byteLength\":6}]," + + "\"accessors\":[" + + "{\"bufferView\":0,\"componentType\":5126,\"count\":3,\"type\":\"VEC3\"}," + + "{\"bufferView\":1,\"componentType\":5123,\"count\":3,\"type\":\"SCALAR\"}]," + + "\"meshes\":[{\"primitives\":[{" + + "\"attributes\":{\"POSITION\":0},\"indices\":1,\"mode\":4}]}]" + + "}"; + return json.getBytes(StandardCharsets.UTF_8); + } + + @Test + void gltfModelRetainsBytesAndLazilyParsesGeometry() { + byte[] data = triangleGltf(); + ARModel model = ARModel.fromGltf(data); + assertArrayEquals(data, model.getGltfBytes()); + // Defensive copy: mutating the returned bytes must not corrupt the model. + model.getGltfBytes()[0] = 0; + + Mesh mesh = model.getMesh(); + assertNotNull(mesh); + assertEquals(3, mesh.getVertices().getVertexCount()); + assertSame(mesh, model.getMesh(), "parsing happens once"); + assertNull(model.getBaseColorImage()); + assertEquals(0xffffffff, model.getColor()); + } + + @Test + void gltfModelFromStream() throws IOException { + ARModel model = ARModel.fromGltf(new ByteArrayInputStream(triangleGltf())); + assertEquals(3, model.getMesh().getVertices().getVertexCount()); + } + + @Test + void meshModelExposesMeshAndColor() { + Mesh sphere = Primitives.sphere(0.5f, 4, 6, false); + ARModel model = ARModel.fromMesh(sphere, 0xffff0000); + assertSame(sphere, model.getMesh()); + assertEquals(0xffff0000, model.getColor()); + assertNull(model.getGltfBytes()); + assertNull(model.getBaseColorImage()); + } + + @Test + void factoriesRejectInvalidArguments() { + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + ARModel.fromGltf((byte[]) null); + } + }); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + ARModel.fromGltf(new byte[0]); + } + }); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + ARModel.fromMesh(null, 0xffffffff); + } + }); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + ARModel.fromMesh(Primitives.sphere(1f, 4, 6, false), null); + } + }); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ar/ARPoseTest.java b/maven/core-unittests/src/test/java/com/codename1/ar/ARPoseTest.java new file mode 100644 index 00000000000..e594ef0fef3 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ar/ARPoseTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.gpu.Matrix4; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for the {@link ARPose} rigid-transform math: matrix conversion in + * both directions, composition and point transformation. Pure math, no + * Display required. Results are cross-checked against {@link Matrix4}. + */ +class ARPoseTest { + + private static final float EPS = 1e-5f; + + @Test + void identityMapsToIdentityMatrix() { + assertArrayEquals(Matrix4.identity(), ARPose.IDENTITY.toMatrix()); + } + + @Test + void pureTranslationMatrix() { + ARPose p = new ARPose(1, 2, 3, 0, 0, 0, 1); + float[] m = p.toMatrix(); + assertArrayEquals(Matrix4.translation(1, 2, 3), m); + } + + @Test + void rotationMatchesMatrix4Rotation() { + // 90 degrees around Y: quaternion (0, sin45, 0, cos45). + float s = (float) Math.sin(Math.PI / 4); + float c = (float) Math.cos(Math.PI / 4); + ARPose p = new ARPose(0, 0, 0, 0, s, 0, c); + float[] expected = Matrix4.rotation((float) Math.PI / 2, 0, 1, 0); + float[] actual = p.toMatrix(); + for (int i = 0; i < 16; i++) { + assertEquals(expected[i], actual[i], EPS, "element " + i); + } + } + + @Test + void toMatrixAndFromMatrixRoundTrip() { + float s = (float) Math.sin(0.6); + float c = (float) Math.cos(0.6); + ARPose original = new ARPose(0.5f, -1.25f, 2f, s * 0.267f, s * 0.535f, s * 0.802f, c); + ARPose restored = ARPose.fromMatrix(original.toMatrix()); + assertEquals(original.getTx(), restored.getTx(), EPS); + assertEquals(original.getTy(), restored.getTy(), EPS); + assertEquals(original.getTz(), restored.getTz(), EPS); + // q and -q are the same rotation; compare via the matrices. + float[] m1 = original.toMatrix(); + float[] m2 = restored.toMatrix(); + for (int i = 0; i < 16; i++) { + assertEquals(m1[i], m2[i], 1e-4f, "element " + i); + } + } + + @Test + void transformComposesLikeMatrixMultiplication() { + float s = (float) Math.sin(Math.PI / 4); + float c = (float) Math.cos(Math.PI / 4); + ARPose a = new ARPose(1, 0, 0, 0, s, 0, c); + ARPose b = new ARPose(0, 2, 0, s, 0, 0, c); + ARPose ab = a.transform(b); + + float[] expected = new float[16]; + Matrix4.multiply(a.toMatrix(), b.toMatrix(), expected); + float[] actual = ab.toMatrix(); + for (int i = 0; i < 16; i++) { + assertEquals(expected[i], actual[i], EPS, "element " + i); + } + } + + @Test + void transformPointRotatesThenTranslates() { + // 90 degrees around Z maps +X to +Y, then translate by (10, 0, 0). + float s = (float) Math.sin(Math.PI / 4); + float c = (float) Math.cos(Math.PI / 4); + ARPose p = new ARPose(10, 0, 0, 0, 0, s, c); + float[] pt = {1, 0, 0}; + p.transformPoint(pt); + assertEquals(10f, pt[0], EPS); + assertEquals(1f, pt[1], EPS); + assertEquals(0f, pt[2], EPS); + } + + @Test + void constructorNormalizesTheQuaternion() { + ARPose p = new ARPose(0, 0, 0, 0, 0, 0, 2); + assertEquals(1f, p.getQw(), EPS); + // Zero quaternion falls back to identity. + ARPose z = new ARPose(0, 0, 0, 0, 0, 0, 0); + assertEquals(1f, z.getQw(), EPS); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ar/ARSessionTest.java b/maven/core-unittests/src/test/java/com/codename1/ar/ARSessionTest.java new file mode 100644 index 00000000000..3d4c174136f --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ar/ARSessionTest.java @@ -0,0 +1,528 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.junit.UITestBase; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for {@link ARSession}: view creation, lifecycle forwarding, hit + * testing, anchor management, node forwarding, and the event bridge that + * marshals implementation events to the EDT with per-id coalescing. + */ +class ARSessionTest extends UITestBase { + + private RecordingARImpl impl; + private ARSession session; + + private final List planeEvents = new ArrayList(); + private final List anchorEvents = new ArrayList(); + private final List trackingStates = new ArrayList(); + + @AfterEach + void closeSession() { + if (session != null) { + session.close(); + session = null; + } + planeEvents.clear(); + anchorEvents.clear(); + trackingStates.clear(); + } + + private ARSession open() { + impl = new RecordingARImpl(); + implementation.setARImpl(impl); + session = AR.open(new ARSessionOptions()); + return session; + } + + private void listenAll() { + session.addPlaneListener(new ARPlaneListener() { + public void planeChanged(ARPlaneEvent ev) { + planeEvents.add(ev); + } + }); + session.addAnchorListener(new ARAnchorListener() { + public void anchorChanged(ARAnchorEvent ev) { + anchorEvents.add(ev); + } + }); + session.addTrackingListener(new ARTrackingListener() { + public void trackingStateChanged(ARSession s, ARTrackingState st, + ARTrackingFailureReason r) { + trackingStates.add(st); + } + }); + } + + private static ARPlane plane(String id) { + return new ARPlane(id, ARPlane.Type.HORIZONTAL_UP, ARPose.IDENTITY, 1f, 1f, null, + ARTrackingState.TRACKING); + } + + // ---- view ---- + + @Test + void createViewIsCachedAndBacklinked() { + open(); + ARView v1 = session.createView(); + ARView v2 = session.createView(); + assertSame(v1, v2); + assertSame(session, v1.getSession()); + assertEquals(1, impl.viewPeerCount); + } + + // ---- lifecycle ---- + + @Test + void pauseResumeForwardOnce() { + open(); + session.pause(); + session.resume(); + assertEquals(1, impl.pauseCount); + assertEquals(1, impl.resumeCount); + } + + @Test + void closeIsIdempotentAndForwardsOnce() { + open(); + session.close(); + session.close(); + assertTrue(session.isClosed()); + assertEquals(1, impl.closeCount); + } + + @Test + void pauseResumeAfterCloseAreDropped() { + open(); + session.close(); + session.pause(); + session.resume(); + assertEquals(0, impl.pauseCount); + assertEquals(0, impl.resumeCount); + } + + // ---- hit testing ---- + + @Test + void hitTestResolvesWithSessionAttachedResults() { + open(); + final ARHitResult hit = new ARHitResult(ARPose.IDENTITY, 1.5f, + ARHitResult.Type.PLANE, plane("p1"), "native-token"); + impl.hitResults = new ARHitResult[]{hit}; + final AtomicReference got = new AtomicReference(); + session.hitTest(0.25f, 0.75f).ready(new com.codename1.util.SuccessCallback() { + public void onSucess(ARHitResult[] hits) { + got.set(hits); + } + }); + flushSerialCalls(); + assertNotNull(got.get()); + assertEquals(1, got.get().length); + assertEquals(0.25f, impl.lastHitX, 0f); + assertEquals(0.75f, impl.lastHitY, 0f); + + // The session attached itself, so createAnchor() works and reaches + // the impl with the native token. + ARAnchor anchor = got.get()[0].createAnchor(); + assertNotNull(anchor); + assertEquals("native-token", impl.lastHitHandle); + assertEquals(1, session.getAnchors().length); + } + + @Test + void hitTestMapsNullResultsToEmptyArray() { + open(); + impl.hitResults = null; + final AtomicReference got = new AtomicReference(); + session.hitTest(0.5f, 0.5f).ready(new com.codename1.util.SuccessCallback() { + public void onSucess(ARHitResult[] hits) { + got.set(hits); + } + }); + flushSerialCalls(); + assertNotNull(got.get()); + assertEquals(0, got.get().length); + } + + @Test + void hitTestAfterCloseThrows() { + open(); + session.close(); + assertThrows(IllegalStateException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + session.hitTest(0.5f, 0.5f); + } + }); + } + + @Test + void unattachedHitResultCannotCreateAnchor() { + final ARHitResult hit = new ARHitResult(ARPose.IDENTITY, 1f, + ARHitResult.Type.FEATURE_POINT, null, null); + assertThrows(IllegalStateException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + hit.createAnchor(); + } + }); + } + + // ---- anchors ---- + + @Test + void createAnchorRegistersAndFiresAdded() { + open(); + listenAll(); + ARAnchor a = session.createAnchor(new ARPose(1, 2, 3, 0, 0, 0, 1)); + assertEquals(1, session.getAnchors().length); + assertSame(a, session.getAnchors()[0]); + assertEquals(1, anchorEvents.size()); + assertEquals(ARAnchorEvent.Kind.ADDED, anchorEvents.get(0).getKind()); + assertSame(a, anchorEvents.get(0).getAnchor()); + assertEquals(1f, a.getPose().getTx(), 0f); + } + + @Test + void createAnchorRejectsNullPose() { + open(); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + session.createAnchor(null); + } + }); + } + + @Test + void detachRemovesAnchorAndFiresRemoved() { + open(); + listenAll(); + ARAnchor a = session.createAnchor(ARPose.IDENTITY); + anchorEvents.clear(); + a.detach(); + assertTrue(a.isDetached()); + assertEquals(0, session.getAnchors().length); + assertEquals(1, impl.removedAnchorIds.size()); + assertEquals(a.getId(), impl.removedAnchorIds.get(0)); + assertEquals(1, anchorEvents.size()); + assertEquals(ARAnchorEvent.Kind.REMOVED, anchorEvents.get(0).getKind()); + // Idempotent. + a.detach(); + assertEquals(1, impl.removedAnchorIds.size()); + } + + @Test + void setNodeForwardsToImplAndDetachedAnchorRejectsIt() { + open(); + ARAnchor a = session.createAnchor(ARPose.IDENTITY); + final ARNode node = new ARNode(ARModel.fromMesh( + com.codename1.gpu.Primitives.sphere(0.1f, 4, 6, false), 0xff00ff00)); + a.setNode(node); + assertEquals(1, impl.setNodeCount); + assertEquals(a.getId(), impl.lastNodeAnchorId); + assertSame(node, impl.lastNode); + assertSame(node, a.getNode()); + + a.setNode(null); + assertEquals(2, impl.setNodeCount); + assertNull(impl.lastNode); + + a.detach(); + assertThrows(IllegalStateException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + a.setNode(node); + } + }); + } + + @Test + void nodeMutationsForwardTheRootSubtree() { + open(); + ARAnchor a = session.createAnchor(ARPose.IDENTITY); + ARNode root = new ARNode(); + a.setNode(root); + impl.nodeChangedCount = 0; + + root.setLocalPosition(0, 0.5f, 0); + assertEquals(1, impl.nodeChangedCount); + assertEquals(a.getId(), impl.lastChangedAnchorId); + assertSame(root, impl.lastChangedNode); + + // Child mutations bubble up to the root. + ARNode child = new ARNode(); + root.addChild(child); + assertEquals(2, impl.nodeChangedCount); + child.setLocalScale(2f); + assertEquals(3, impl.nodeChangedCount); + assertSame(root, impl.lastChangedNode); + + // Detached subtrees stop notifying. + a.setNode(null); + impl.nodeChangedCount = 0; + root.setVisible(false); + assertEquals(0, impl.nodeChangedCount); + } + + @Test + void anchorContentRootMayNotHaveAParent() { + open(); + final ARAnchor a = session.createAnchor(ARPose.IDENTITY); + ARNode parent = new ARNode(); + final ARNode child = new ARNode(); + parent.addChild(child); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + a.setNode(child); + } + }); + } + + // ---- event bridge ---- + + @Test + void planeEventsFromBackgroundThreadArriveAfterFlush() { + open(); + listenAll(); + impl.onBackgroundThread(new Runnable() { + public void run() { + impl.sink.onPlaneAdded(plane("p1")); + } + }); + // Nothing delivered until the EDT drains. + assertEquals(0, planeEvents.size()); + flushSerialCalls(); + assertEquals(1, planeEvents.size()); + assertEquals(ARPlaneEvent.Kind.ADDED, planeEvents.get(0).getKind()); + assertEquals("p1", planeEvents.get(0).getPlane().getId()); + assertEquals(1, session.getPlanes().length); + } + + @Test + void planeUpdatesCoalesceToTheLatestSnapshot() { + open(); + listenAll(); + impl.sink.onPlaneAdded(plane("p1")); + flushSerialCalls(); + planeEvents.clear(); + + final ARPlane last = new ARPlane("p1", ARPlane.Type.HORIZONTAL_UP, ARPose.IDENTITY, + 5f, 5f, null, ARTrackingState.TRACKING); + impl.onBackgroundThread(new Runnable() { + public void run() { + impl.sink.onPlaneUpdated(plane("p1")); + impl.sink.onPlaneUpdated(last); + } + }); + flushSerialCalls(); + assertEquals(1, planeEvents.size(), "two updates for one id coalesce into one event"); + assertEquals(ARPlaneEvent.Kind.UPDATED, planeEvents.get(0).getKind()); + assertEquals(5f, session.getPlanes()[0].getExtentX(), 0f); + } + + @Test + void planeRemovalDropsThePlaneAndSkipsStaleUpdates() { + open(); + listenAll(); + impl.sink.onPlaneAdded(plane("p1")); + flushSerialCalls(); + planeEvents.clear(); + + impl.sink.onPlaneRemoved("p1"); + impl.sink.onPlaneUpdated(plane("p1")); + flushSerialCalls(); + assertEquals(1, planeEvents.size(), "the stale update after removal is dropped"); + assertEquals(ARPlaneEvent.Kind.REMOVED, planeEvents.get(0).getKind()); + assertEquals(0, session.getPlanes().length); + } + + @Test + void updateBeforeAddInTheSameBatchAppliesAfterTheAdd() { + open(); + listenAll(); + final ARPlane refined = new ARPlane("p1", ARPlane.Type.HORIZONTAL_UP, ARPose.IDENTITY, + 3f, 3f, null, ARTrackingState.TRACKING); + impl.sink.onPlaneUpdated(refined); + impl.sink.onPlaneAdded(plane("p1")); + flushSerialCalls(); + // Ordered add applies first, then the coalesced refinement. + assertEquals(2, planeEvents.size()); + assertEquals(ARPlaneEvent.Kind.ADDED, planeEvents.get(0).getKind()); + assertEquals(ARPlaneEvent.Kind.UPDATED, planeEvents.get(1).getKind()); + assertEquals(3f, session.getPlanes()[0].getExtentX(), 0f); + } + + @Test + void platformAnchorsArriveAsSubtypes() { + open(); + listenAll(); + impl.sink.onAnchorAdded(new ARImageAnchor("img-1", ARPose.IDENTITY, "poster", 0.4f)); + impl.sink.onAnchorAdded(new ARFaceAnchor("face-1", ARPose.IDENTITY)); + flushSerialCalls(); + assertEquals(2, anchorEvents.size()); + assertTrue(anchorEvents.get(0).getAnchor() instanceof ARImageAnchor); + assertEquals("poster", + ((ARImageAnchor) anchorEvents.get(0).getAnchor()).getReferenceImageName()); + assertTrue(anchorEvents.get(1).getAnchor() instanceof ARFaceAnchor); + assertEquals(2, session.getAnchors().length); + } + + @Test + void anchorUpdatesRefreshPoseAndState() { + open(); + listenAll(); + ARAnchor a = session.createAnchor(ARPose.IDENTITY); + anchorEvents.clear(); + impl.sink.onAnchorUpdated(a.getId(), new ARPose(0, 1, 0, 0, 0, 0, 1), + ARTrackingState.LIMITED); + flushSerialCalls(); + assertEquals(1, anchorEvents.size()); + assertEquals(ARAnchorEvent.Kind.UPDATED, anchorEvents.get(0).getKind()); + assertEquals(1f, a.getPose().getTy(), 0f); + assertEquals(ARTrackingState.LIMITED, a.getTrackingState()); + } + + @Test + void faceUpdatesCarryGeometryAndMergeWithPoseUpdates() { + open(); + listenAll(); + impl.sink.onAnchorAdded(new ARFaceAnchor("face-1", ARPose.IDENTITY)); + flushSerialCalls(); + ARFaceAnchor face = (ARFaceAnchor) session.getAnchors()[0]; + + final ARPose[] regions = new ARPose[ARFaceRegion.values().length]; + regions[ARFaceRegion.NOSE_TIP.ordinal()] = new ARPose(0, 0, 0.1f, 0, 0, 0, 1); + final float[] mesh = {0, 0, 0, 1, 0, 0, 0, 1, 0}; + final int[] tris = {0, 1, 2}; + // A face payload followed by a pose-only refinement in the same batch + // must not lose the geometry. + impl.sink.onFaceAnchorUpdated("face-1", null, null, regions, mesh, tris); + impl.sink.onAnchorUpdated("face-1", new ARPose(0, 2, 0, 0, 0, 0, 1), + ARTrackingState.TRACKING); + flushSerialCalls(); + + assertEquals(2f, face.getPose().getTy(), 0f); + assertNotNull(face.getRegionPose(ARFaceRegion.NOSE_TIP)); + assertNull(face.getRegionPose(ARFaceRegion.LEFT_EYE)); + assertArrayEquals(mesh, face.getMeshVertices()); + assertArrayEquals(tris, face.getMeshTriangles()); + } + + @Test + void platformAnchorRemovalDetachesAndFires() { + open(); + listenAll(); + impl.sink.onAnchorAdded(new ARImageAnchor("img-1", ARPose.IDENTITY, "poster", 0.4f)); + flushSerialCalls(); + ARAnchor a = session.getAnchors()[0]; + anchorEvents.clear(); + impl.sink.onAnchorRemoved("img-1"); + flushSerialCalls(); + assertTrue(a.isDetached()); + assertEquals(0, session.getAnchors().length); + assertEquals(ARAnchorEvent.Kind.REMOVED, anchorEvents.get(0).getKind()); + } + + @Test + void trackingStateChangesFireAndCache() { + open(); + listenAll(); + assertEquals(ARTrackingState.NOT_TRACKING, session.getTrackingState()); + impl.sink.onTrackingStateChanged(ARTrackingState.TRACKING, ARTrackingFailureReason.NONE); + flushSerialCalls(); + assertEquals(1, trackingStates.size()); + assertEquals(ARTrackingState.TRACKING, session.getTrackingState()); + assertEquals(ARTrackingFailureReason.NONE, session.getTrackingFailureReason()); + + impl.sink.onTrackingStateChanged(ARTrackingState.LIMITED, + ARTrackingFailureReason.EXCESSIVE_MOTION); + flushSerialCalls(); + assertEquals(ARTrackingFailureReason.EXCESSIVE_MOTION, + session.getTrackingFailureReason()); + } + + @Test + void cameraPoseAndLightEstimateAreCachedForPolling() { + open(); + assertEquals(ARPose.IDENTITY, session.getCameraPose()); + assertFalse(session.getLightEstimate().isValid()); + + impl.sink.onCameraPose(new ARPose(1, 1, 1, 0, 0, 0, 1)); + impl.sink.onCameraPose(new ARPose(2, 2, 2, 0, 0, 0, 1)); + impl.sink.onLightEstimate(new ARLightEstimate(true, 0.5f, 1f, 0.9f, 0.8f)); + flushSerialCalls(); + // Coalesced: only the latest pose survives. + assertEquals(2f, session.getCameraPose().getTx(), 0f); + assertTrue(session.getLightEstimate().isValid()); + assertEquals(0.5f, session.getLightEstimate().getAmbientIntensity(), 0f); + assertEquals(0.9f, session.getLightEstimate().getColorCorrection()[1], 0f); + } + + @Test + void eventsAfterCloseAreDropped() { + open(); + listenAll(); + ARImpl_EventSinkHolder holder = new ARImpl_EventSinkHolder(impl.sink); + session.close(); + holder.sink.onPlaneAdded(plane("p1")); + holder.sink.onTrackingStateChanged(ARTrackingState.TRACKING, + ARTrackingFailureReason.NONE); + flushSerialCalls(); + assertEquals(0, planeEvents.size()); + assertEquals(0, trackingStates.size()); + } + + /** Tiny holder so the closed-session test reads clearly. */ + private static final class ARImpl_EventSinkHolder { + final com.codename1.impl.ARImpl.EventSink sink; + + ARImpl_EventSinkHolder(com.codename1.impl.ARImpl.EventSink sink) { + this.sink = sink; + } + } + + @Test + void listenersCanBeRemoved() { + open(); + final int[] count = {0}; + ARPlaneListener l = new ARPlaneListener() { + public void planeChanged(ARPlaneEvent ev) { + count[0]++; + } + }; + session.addPlaneListener(l); + impl.sink.onPlaneAdded(plane("p1")); + flushSerialCalls(); + assertEquals(1, count[0]); + session.removePlaneListener(l); + impl.sink.onPlaneAdded(plane("p2")); + flushSerialCalls(); + assertEquals(1, count[0]); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ar/ARTest.java b/maven/core-unittests/src/test/java/com/codename1/ar/ARTest.java new file mode 100644 index 00000000000..94436df955c --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ar/ARTest.java @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.junit.UITestBase; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for the static {@link AR} entry point: support probing, + * capabilities, the single-open-session contract and the permission request. + * Drives a hand-written {@link RecordingARImpl} installed through the test + * implementation. + */ +class ARTest extends UITestBase { + + private ARSession opened; + + @AfterEach + void closeAnyOpenSession() { + if (opened != null) { + opened.close(); + opened = null; + } + } + + private RecordingARImpl install() { + RecordingARImpl impl = new RecordingARImpl(); + implementation.setARImpl(impl); + return impl; + } + + @Test + void notSupportedWithoutBackend() { + implementation.setARImpl(null); + assertFalse(AR.isSupported()); + } + + @Test + void supportedWhenBackendPresent() { + install(); + assertTrue(AR.isSupported()); + } + + @Test + void capabilitiesAllFalseWithoutBackend() { + implementation.setARImpl(null); + ARCapabilities caps = AR.getCapabilities(); + assertFalse(caps.isWorldTrackingSupported()); + assertFalse(caps.isPlaneDetectionSupported()); + assertFalse(caps.isImageTrackingSupported()); + assertFalse(caps.isFaceTrackingSupported()); + assertFalse(caps.isLightEstimationSupported()); + } + + @Test + void capabilitiesComeFromBackendAndProbeIsClosed() { + RecordingARImpl impl = install(); + impl.capabilities = new ARCapabilities(true, true, false, false, true); + ARCapabilities caps = AR.getCapabilities(); + assertTrue(caps.isWorldTrackingSupported()); + assertFalse(caps.isImageTrackingSupported()); + assertEquals(1, impl.closeCount); + } + + @Test + void nullBackendCapabilitiesMapToUnsupported() { + RecordingARImpl impl = install(); + impl.capabilitiesReturnsNull = true; + assertFalse(AR.getCapabilities().isWorldTrackingSupported()); + } + + @Test + void openThrowsWhenUnsupported() { + implementation.setARImpl(null); + assertThrows(IllegalStateException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + AR.open(new ARSessionOptions()); + } + }); + } + + @Test + void openSucceedsAndPassesOptionsToBackend() { + RecordingARImpl impl = install(); + ARSessionOptions opts = new ARSessionOptions() + .planeDetection(ARPlaneDetection.HORIZONTAL_AND_VERTICAL); + opened = AR.open(opts); + assertNotNull(opened); + assertFalse(opened.isClosed()); + assertSame(opts, impl.openedOptions); + assertEquals(1, impl.openCount); + assertNotNull(impl.sink); + } + + @Test + void openWithNullOptionsUsesDefaults() { + install(); + opened = AR.open(null); + assertNotNull(opened.getOptions()); + assertEquals(ARTrackingMode.WORLD, opened.getOptions().getTrackingMode()); + assertEquals(ARPlaneDetection.HORIZONTAL, opened.getOptions().getPlaneDetection()); + assertTrue(opened.getOptions().isLightEstimation()); + } + + @Test + void secondOpenWhileActiveThrows() { + install(); + opened = AR.open(null); + assertThrows(IllegalStateException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + AR.open(null); + } + }); + } + + @Test + void openAfterClosingPreviousSucceeds() { + install(); + ARSession first = AR.open(null); + first.close(); + opened = AR.open(null); + assertFalse(opened.isClosed()); + } + + @Test + void openWrapsBackendIOExceptionAndClosesImpl() { + RecordingARImpl impl = install(); + impl.openFailure = new IOException("no camera"); + try { + AR.open(null); + fail("expected RuntimeException"); + } catch (RuntimeException expected) { + assertNotNull(expected.getMessage()); + } + assertEquals(1, impl.closeCount); + // The failed session did not occupy the active slot. + impl.openFailure = null; + opened = AR.open(null); + assertNotNull(opened); + } + + @Test + void requestPermissionsDeliversFalseWithoutBackend() { + implementation.setARImpl(null); + assertEquals(Boolean.FALSE, awaitPermission()); + } + + @Test + void requestPermissionsDeliversBackendResultAndClosesProbe() { + RecordingARImpl impl = install(); + assertEquals(Boolean.TRUE, awaitPermission()); + assertEquals(1, impl.closeCount); + } + + @Test + void requestPermissionsDeliversFalseOnBackendError() { + RecordingARImpl impl = install(); + impl.permissionFailure = new RuntimeException("denied"); + assertEquals(Boolean.FALSE, awaitPermission()); + assertEquals(1, impl.closeCount); + } + + @Test + void requestPermissionsToleratesNullCallback() { + install(); + AR.requestPermissions(null); + flushSerialCalls(); + } + + private Boolean awaitPermission() { + final AtomicReference result = new AtomicReference(); + AR.requestPermissions(new SuccessCallback() { + public void onSucess(Boolean value) { + result.set(value); + } + }); + int budget = 4000; + while (result.get() == null && budget > 0) { + flushSerialCalls(); + budget -= 5; + } + return result.get(); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ar/RecordingARImpl.java b/maven/core-unittests/src/test/java/com/codename1/ar/RecordingARImpl.java new file mode 100644 index 00000000000..6612bc69231 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ar/RecordingARImpl.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ar; + +import com.codename1.impl.ARImpl; +import com.codename1.ui.PeerComponent; +import com.codename1.util.AsyncResource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Hand-written recording {@link ARImpl} for the AR unit tests. Counts + * lifecycle calls, captures the last arguments and exposes the installed + * {@link ARImpl.EventSink} through fire helpers so tests can push platform + * events - optionally from a background thread to prove EDT marshaling. + */ +class RecordingARImpl extends ARImpl { + + ARCapabilities capabilities = new ARCapabilities(true, true, true, true, true); + boolean capabilitiesReturnsNull; + + IOException openFailure; + ARSessionOptions openedOptions; + int openCount; + int closeCount; + int pauseCount; + int resumeCount; + int viewPeerCount; + + EventSink sink; + + ARHitResult[] hitResults = new ARHitResult[0]; + float lastHitX = Float.NaN; + float lastHitY = Float.NaN; + + int anchorSeq; + final List createdAnchorIds = new ArrayList(); + final List removedAnchorIds = new ArrayList(); + Object lastHitHandle; + + String lastNodeAnchorId; + ARNode lastNode; + int setNodeCount; + String lastChangedAnchorId; + ARNode lastChangedNode; + int nodeChangedCount; + + Boolean permissionResult = Boolean.TRUE; + Throwable permissionFailure; + + @Override + public ARCapabilities getCapabilities() { + return capabilitiesReturnsNull ? null : capabilities; + } + + @Override + public void setEventSink(EventSink sink) { + this.sink = sink; + } + + @Override + public void open(ARSessionOptions opts) throws IOException { + openCount++; + openedOptions = opts; + if (openFailure != null) { + throw openFailure; + } + } + + @Override + public PeerComponent createViewPeer() { + viewPeerCount++; + return null; + } + + @Override + public void hitTest(float xNorm, float yNorm, AsyncResource result) { + lastHitX = xNorm; + lastHitY = yNorm; + result.complete(hitResults); + } + + @Override + public String createAnchor(ARPose pose) { + String id = "anchor-" + (++anchorSeq); + createdAnchorIds.add(id); + return id; + } + + @Override + public String createAnchorFromHit(Object nativeHandle, ARPose pose) { + lastHitHandle = nativeHandle; + String id = "hit-anchor-" + (++anchorSeq); + createdAnchorIds.add(id); + return id; + } + + @Override + public void removeAnchor(String anchorId) { + removedAnchorIds.add(anchorId); + } + + @Override + public void setAnchorNode(String anchorId, ARNode node) { + setNodeCount++; + lastNodeAnchorId = anchorId; + lastNode = node; + } + + @Override + public void nodeChanged(String anchorId, ARNode node) { + nodeChangedCount++; + lastChangedAnchorId = anchorId; + lastChangedNode = node; + } + + @Override + public void requestPermissions(AsyncResource result) { + if (permissionFailure != null) { + result.error(permissionFailure); + } else { + result.complete(permissionResult); + } + } + + @Override + public void pause() { + pauseCount++; + } + + @Override + public void resume() { + resumeCount++; + } + + @Override + public void close() { + closeCount++; + } + + /** + * Runs {@code r} on a freshly started background thread and joins it, so + * tests can prove sink events arriving off the EDT are marshaled. + */ + void onBackgroundThread(Runnable r) { + Thread t = new Thread(r, "ar-test-events"); + t.start(); + try { + t.join(5000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/gpu/GltfLoaderTest.java b/maven/core-unittests/src/test/java/com/codename1/gpu/GltfLoaderTest.java index 397204360de..47019d1e485 100644 --- a/maven/core-unittests/src/test/java/com/codename1/gpu/GltfLoaderTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/gpu/GltfLoaderTest.java @@ -243,6 +243,42 @@ void synthesizesSequentialIndicesWhenPrimitiveHasNone() { assertEquals(2, idx[2]); } + @Test + void deviceFreeLoadMatchesDeviceLoad() { + byte[] data = utf8(triangleGltfJson(triangleBuffer())); + Mesh direct = GltfLoader.load(data); + Mesh viaDevice = GltfLoader.load(new StubDevice(), data); + assertEquals(viaDevice.getVertices().getVertexCount(), direct.getVertices().getVertexCount()); + assertEquals(viaDevice.getIndices().getIndexCount(), direct.getIndices().getIndexCount()); + assertArrayEquals(viaDevice.getVertices().getData(), direct.getVertices().getData()); + assertArrayEquals(viaDevice.getIndices().getData(), direct.getIndices().getData()); + } + + @Test + void deviceFreeStreamLoadClosesTheStream() throws IOException { + byte[] data = utf8(triangleGltfJson(triangleBuffer())); + final boolean[] closed = {false}; + InputStream in = new ByteArrayInputStream(data) { + @Override + public void close() throws IOException { + closed[0] = true; + super.close(); + } + }; + Mesh mesh = GltfLoader.load(in); + assertNotNull(mesh); + assertTrue(closed[0], "loader must close the supplied stream"); + } + + @Test + void imageModelWithoutMaterialHasMeshAndNullImage() { + byte[] data = utf8(triangleGltfJson(triangleBuffer())); + GltfLoader.GltfImageModel model = GltfLoader.loadImageModel(data); + assertNotNull(model.getMesh()); + assertEquals(3, model.getMesh().getVertices().getVertexCount()); + assertNull(model.getBaseColorImage()); + } + @Test void rejectsEmptyOrTooShortData() { StubDevice d = new StubDevice(); diff --git a/maven/core-unittests/src/test/java/com/codename1/gpu/PrimitivesTest.java b/maven/core-unittests/src/test/java/com/codename1/gpu/PrimitivesTest.java index 55d17e31bf0..d223f99e245 100644 --- a/maven/core-unittests/src/test/java/com/codename1/gpu/PrimitivesTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/gpu/PrimitivesTest.java @@ -36,8 +36,9 @@ class PrimitivesTest { /** Minimal device: only the abstract members are stubbed; the concrete - * {@code createVertexBuffer}/{@code createIndexBuffer} are inherited. */ - private static final class HeadlessDevice extends GraphicsDevice { + * {@code createVertexBuffer}/{@code createIndexBuffer} are inherited. + * Package-visible so sibling primitive tests can reuse it. */ + static final class HeadlessDevice extends GraphicsDevice { public GpuCapabilities getCapabilities() { return null; } diff --git a/maven/core-unittests/src/test/java/com/codename1/gpu/QuaternionTest.java b/maven/core-unittests/src/test/java/com/codename1/gpu/QuaternionTest.java new file mode 100644 index 00000000000..d51613ebcb3 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/gpu/QuaternionTest.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.gpu; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for the {@link Quaternion} float[4] utilities. Pure math, so no + * Display/platform setup is required. Results are cross-checked against + * {@link Matrix4} where both APIs express the same rotation. + */ +class QuaternionTest { + + private static final float EPS = 1e-5f; + + private static void assertQuatEquals(float[] expected, float[] actual, float eps) { + // q and -q encode the same rotation; normalize the sign before comparing. + float sign = (expected[0] * actual[0] + expected[1] * actual[1] + + expected[2] * actual[2] + expected[3] * actual[3]) < 0 ? -1f : 1f; + for (int i = 0; i < 4; i++) { + assertEquals(expected[i], sign * actual[i], eps, "component " + i); + } + } + + @Test + void identityIsNoRotation() { + float[] q = Quaternion.identity(); + assertArrayEquals(new float[]{0, 0, 0, 1}, q); + + float[] v = {1.5f, -2.5f, 3.5f}; + Quaternion.rotateVector(q, v); + assertArrayEquals(new float[]{1.5f, -2.5f, 3.5f}, v); + + float[] m = new float[16]; + Quaternion.toMatrix(q, m); + assertArrayEquals(Matrix4.identity(), m); + } + + @Test + void toMatrixMatchesMatrix4RotationForArbitraryAxes() { + float[][] axes = {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}, {1, 2, 3}, {-1, 0.5f, 2}}; + float[] angles = {0.3f, 1.1f, (float) Math.PI / 2, 2.7f, -0.9f}; + float[] qm = new float[16]; + for (int i = 0; i < axes.length; i++) { + float[] q = Quaternion.fromAxisAngle(angles[i], axes[i][0], axes[i][1], axes[i][2]); + Quaternion.toMatrix(q, qm); + float[] mm = Matrix4.rotation(angles[i], axes[i][0], axes[i][1], axes[i][2]); + for (int c = 0; c < 16; c++) { + assertEquals(mm[c], qm[c], EPS, "axis " + i + " element " + c); + } + } + } + + @Test + void zeroAxisProducesIdentity() { + float[] q = Quaternion.fromAxisAngle(1.0f, 0, 0, 0); + assertArrayEquals(new float[]{0, 0, 0, 1}, q); + } + + @Test + void multiplyComposesLikeMatrixMultiplication() { + float[] a = Quaternion.fromAxisAngle((float) Math.PI / 2, 0, 0, 1); + float[] b = Quaternion.fromAxisAngle((float) Math.PI / 3, 1, 0, 0); + float[] ab = new float[4]; + Quaternion.multiply(a, b, ab); + + float[] ma = Matrix4.rotation((float) Math.PI / 2, 0, 0, 1); + float[] mb = Matrix4.rotation((float) Math.PI / 3, 1, 0, 0); + float[] mab = new float[16]; + Matrix4.multiply(ma, mb, mab); + + float[] qm = new float[16]; + Quaternion.toMatrix(ab, qm); + for (int c = 0; c < 16; c++) { + assertEquals(mab[c], qm[c], EPS, "element " + c); + } + } + + @Test + void multiplyToleratesAliasedDestination() { + float[] a = Quaternion.fromAxisAngle(0.7f, 0, 1, 0); + float[] b = Quaternion.fromAxisAngle(0.4f, 1, 0, 0); + float[] expected = new float[4]; + Quaternion.multiply(a, b, expected); + Quaternion.multiply(a, b, a); + assertArrayEquals(expected, a); + } + + @Test + void rotateVectorKnownCase() { + // 90 degrees around +Z maps +X to +Y. + float[] q = Quaternion.fromAxisAngle((float) Math.PI / 2, 0, 0, 1); + float[] v = {1, 0, 0}; + Quaternion.rotateVector(q, v); + assertEquals(0f, v[0], EPS); + assertEquals(1f, v[1], EPS); + assertEquals(0f, v[2], EPS); + } + + @Test + void conjugateUndoesRotation() { + float[] q = Quaternion.fromAxisAngle(1.3f, 2, -1, 0.5f); + float[] inv = new float[4]; + Quaternion.conjugate(q, inv); + float[] v = {0.4f, -1.2f, 2.2f}; + Quaternion.rotateVector(q, v); + Quaternion.rotateVector(inv, v); + assertEquals(0.4f, v[0], EPS); + assertEquals(-1.2f, v[1], EPS); + assertEquals(2.2f, v[2], EPS); + } + + @Test + void normalizeRestoresUnitLengthAndResetsZero() { + float[] q = {2, 0, 0, 2}; + Quaternion.normalize(q); + float len = (float) Math.sqrt(q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]); + assertEquals(1f, len, EPS); + + float[] zero = {0, 0, 0, 0}; + Quaternion.normalize(zero); + assertArrayEquals(new float[]{0, 0, 0, 1}, zero); + } + + @Test + void slerpEndpointsAndMidpoint() { + float[] a = Quaternion.identity(); + float[] b = Quaternion.fromAxisAngle((float) Math.PI / 2, 0, 1, 0); + float[] out = new float[4]; + + Quaternion.slerp(a, b, 0f, out); + assertQuatEquals(a, out, EPS); + Quaternion.slerp(a, b, 1f, out); + assertQuatEquals(b, out, EPS); + + // Halfway between identity and a 90 degree yaw is a 45 degree yaw. + Quaternion.slerp(a, b, 0.5f, out); + float[] mid = Quaternion.fromAxisAngle((float) Math.PI / 4, 0, 1, 0); + assertQuatEquals(mid, out, EPS); + } + + @Test + void slerpTakesShortestArc() { + float[] a = Quaternion.fromAxisAngle(0.1f, 0, 0, 1); + // Same rotation as some b but with all components negated; slerp must + // not swing the long way around. + float[] b = Quaternion.fromAxisAngle(0.3f, 0, 0, 1); + float[] negB = {-b[0], -b[1], -b[2], -b[3]}; + float[] out = new float[4]; + Quaternion.slerp(a, negB, 0.5f, out); + float[] mid = Quaternion.fromAxisAngle(0.2f, 0, 0, 1); + assertQuatEquals(mid, out, EPS); + } + + @Test + void integrateGyroConstantRateMatchesAxisAngle() { + // 0.5 rad/s around body X for 100 steps of 10ms = 0.5 radians total. + float[] q = Quaternion.identity(); + for (int i = 0; i < 100; i++) { + Quaternion.integrateGyro(q, 0.5f, 0, 0, 0.01f, q); + } + float[] expected = Quaternion.fromAxisAngle(0.5f, 1, 0, 0); + assertQuatEquals(expected, q, 1e-4f); + } + + @Test + void integrateGyroZeroRateKeepsOrientation() { + float[] q = Quaternion.fromAxisAngle(0.8f, 0, 1, 0); + float[] out = new float[4]; + Quaternion.integrateGyro(q, 0, 0, 0, 0.02f, out); + assertArrayEquals(q, out); + } + + @Test + void integrateGyroStaysNormalizedOverLongRuns() { + float[] q = Quaternion.identity(); + for (int i = 0; i < 10000; i++) { + Quaternion.integrateGyro(q, 0.7f, -0.3f, 0.2f, 0.005f, q); + } + float len = (float) Math.sqrt(q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]); + assertEquals(1f, len, 1e-4f); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/gpu/SpherePrimitiveTest.java b/maven/core-unittests/src/test/java/com/codename1/gpu/SpherePrimitiveTest.java new file mode 100644 index 00000000000..fe2afd517f8 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/gpu/SpherePrimitiveTest.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.gpu; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for {@link Primitives#sphere(float, int, int, boolean)}: vertex and + * index counts, radius, equirectangular UV mapping with seam duplication and + * the inside-out normal/winding flip used by the 360 media viewer. + */ +class SpherePrimitiveTest { + + private static final float EPS = 1e-5f; + + @Test + void vertexAndIndexCountsMatchTessellation() { + int lat = 6; + int lon = 8; + Mesh sphere = Primitives.sphere(2.0f, lat, lon, false); + assertEquals(PrimitiveType.TRIANGLES, sphere.getPrimitiveType()); + assertSame(VertexFormat.POSITION_NORMAL_TEXCOORD, sphere.getVertices().getFormat()); + assertEquals((lat + 1) * (lon + 1), sphere.getVertices().getVertexCount()); + assertEquals(lat * lon * 6, sphere.getIndices().getIndexCount()); + } + + @Test + void allPositionsLieOnTheRadius() { + float radius = 3.5f; + Mesh sphere = Primitives.sphere(radius, 5, 7, false); + float[] v = sphere.getVertices().getData(); + int count = sphere.getVertices().getVertexCount(); + for (int i = 0; i < count; i++) { + int o = i * 8; + float len = (float) Math.sqrt(v[o] * v[o] + v[o + 1] * v[o + 1] + v[o + 2] * v[o + 2]); + assertEquals(radius, len, 1e-4f, "vertex " + i); + } + } + + @Test + void uvsAreEquirectangularWithSeamDuplication() { + int lat = 4; + int lon = 6; + Mesh sphere = Primitives.sphere(1.0f, lat, lon, false); + float[] v = sphere.getVertices().getData(); + int count = sphere.getVertices().getVertexCount(); + for (int i = 0; i < count; i++) { + int o = i * 8; + assertTrue(v[o + 6] >= 0f && v[o + 6] <= 1f, "u out of range at vertex " + i); + assertTrue(v[o + 7] >= 0f && v[o + 7] <= 1f, "v out of range at vertex " + i); + } + // North pole row has v=0, south pole row v=1. + assertEquals(0f, v[7], EPS); + int lastRow = lat * (lon + 1) * 8; + assertEquals(1f, v[lastRow + 7], EPS); + + // The seam duplicates positions: first and last vertex of a row share a + // position but carry u=0 and u=1 respectively. + int row = 2 * (lon + 1) * 8; // a mid-latitude row + int rowEnd = row + lon * 8; + assertEquals(v[row], v[rowEnd], 1e-4f); + assertEquals(v[row + 1], v[rowEnd + 1], 1e-4f); + assertEquals(v[row + 2], v[rowEnd + 2], 1e-4f); + assertEquals(0f, v[row + 6], EPS); + assertEquals(1f, v[rowEnd + 6], EPS); + } + + @Test + void insideOutSphereReversesUSoPanoramasReadCorrectly() { + int lat = 4; + int lon = 6; + float[] in = Primitives.sphere(1.0f, lat, lon, true).getVertices().getData(); + float[] out = Primitives.sphere(1.0f, lat, lon, false).getVertices().getData(); + int row = 2 * (lon + 1) * 8; + // Same positions, mirrored u: viewed from inside the sphere the + // horizontal direction flips, so u runs 1 -> 0 along the row. + assertEquals(1f, in[row + 6], EPS); + assertEquals(0f, in[row + lon * 8 + 6], EPS); + for (int c = 0; c <= lon; c++) { + int o = row + c * 8; + assertEquals(out[o], in[o], 1e-4f, "positions must match"); + assertEquals(1f - out[o + 6], in[o + 6], EPS, "u mirrored at column " + c); + assertEquals(out[o + 7], in[o + 7], EPS, "v unchanged at column " + c); + } + } + + @Test + void outwardNormalsPointAwayFromCenter() { + Mesh sphere = Primitives.sphere(2.0f, 5, 7, false); + assertNormalOrientation(sphere, true); + } + + @Test + void insideOutNormalsPointTowardCenter() { + Mesh sphere = Primitives.sphere(2.0f, 5, 7, true); + assertNormalOrientation(sphere, false); + } + + private static void assertNormalOrientation(Mesh sphere, boolean outward) { + float[] v = sphere.getVertices().getData(); + int count = sphere.getVertices().getVertexCount(); + for (int i = 0; i < count; i++) { + int o = i * 8; + float dot = v[o] * v[o + 3] + v[o + 1] * v[o + 4] + v[o + 2] * v[o + 5]; + // Skip degenerate dot at poles is impossible: position length equals + // the radius everywhere, so |dot| is always the radius. + if (outward) { + assertTrue(dot > 0f, "normal should point outward at vertex " + i); + } else { + assertTrue(dot < 0f, "normal should point inward at vertex " + i); + } + float nlen = (float) Math.sqrt(v[o + 3] * v[o + 3] + v[o + 4] * v[o + 4] + v[o + 5] * v[o + 5]); + assertEquals(1f, nlen, 1e-4f, "normal not unit length at vertex " + i); + } + } + + @Test + void windingFlipsWhenInsideOut() { + // The geometric normal of a mid-latitude triangle must face the same way + // as the vertex normals: away from the center outward, toward it inside + // out. + assertEquals(1, windingSign(Primitives.sphere(1.0f, 6, 8, false))); + assertEquals(-1, windingSign(Primitives.sphere(1.0f, 6, 8, true))); + } + + private static int windingSign(Mesh sphere) { + float[] v = sphere.getVertices().getData(); + short[] idx = sphere.getIndices().getData(); + // First triangle of the second latitude band (away from the degenerate + // pole row). 8 longitude bands -> band size is 8 * 6 indices. + int t = 8 * 6; + int a = (idx[t] & 0xffff) * 8; + int b = (idx[t + 1] & 0xffff) * 8; + int c = (idx[t + 2] & 0xffff) * 8; + float ux = v[b] - v[a]; + float uy = v[b + 1] - v[a + 1]; + float uz = v[b + 2] - v[a + 2]; + float wx = v[c] - v[a]; + float wy = v[c + 1] - v[a + 1]; + float wz = v[c + 2] - v[a + 2]; + float nx = uy * wz - uz * wy; + float ny = uz * wx - ux * wz; + float nz = ux * wy - uy * wx; + // Compare against the direction from the center to the triangle. + float dot = nx * v[a] + ny * v[a + 1] + nz * v[a + 2]; + return dot > 0f ? 1 : -1; + } + + @Test + void deviceOverloadProducesTheSameMesh() { + Mesh direct = Primitives.sphere(1.5f, 4, 6, true); + Mesh viaDevice = Primitives.sphere(new PrimitivesTest.HeadlessDevice(), 1.5f, 4, 6, true); + assertEquals(direct.getVertices().getVertexCount(), viaDevice.getVertices().getVertexCount()); + assertEquals(direct.getIndices().getIndexCount(), viaDevice.getIndices().getIndexCount()); + assertArrayEquals(direct.getVertices().getData(), viaDevice.getVertices().getData()); + assertArrayEquals(direct.getIndices().getData(), viaDevice.getIndices().getData()); + } + + @Test + void rejectsInvalidArguments() { + assertThrows(IllegalArgumentException.class, () -> Primitives.sphere(0f, 4, 6, false)); + assertThrows(IllegalArgumentException.class, () -> Primitives.sphere(-1f, 4, 6, false)); + assertThrows(IllegalArgumentException.class, () -> Primitives.sphere(1f, 1, 6, false)); + assertThrows(IllegalArgumentException.class, () -> Primitives.sphere(1f, 4, 2, false)); + // (lat+1)*(lon+1) must stay within the unsigned short index range. + assertThrows(IllegalArgumentException.class, () -> Primitives.sphere(1f, 300, 300, false)); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/sensors/FakeMotionSensorManager.java b/maven/core-unittests/src/test/java/com/codename1/sensors/FakeMotionSensorManager.java new file mode 100644 index 00000000000..a3888e2fcdf --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/sensors/FakeMotionSensorManager.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.sensors; + +import java.util.HashSet; +import java.util.Set; + +/** + * Deterministic {@link MotionSensorManager} for unit tests. Sensors are + * "supported" per the flags handed to the constructor, native start/stop + * calls are counted, and tests push readings synchronously through + * {@link #fire(int, float, float, float, long)} (delivered via + * {@code Display.callSerially}, so drain with {@code flushSerialCalls()}). + */ +public class FakeMotionSensorManager extends MotionSensorManager { + + private final Set supported = new HashSet(); + public final Set started = new HashSet(); + public int startCount; + public int stopCount; + + public FakeMotionSensorManager(int... supportedTypes) { + for (int t : supportedTypes) { + supported.add(t); + } + } + + @Override + protected boolean isNativeSensorSupported(int type) { + return supported.contains(type); + } + + @Override + protected void startNativeSensor(int type) { + started.add(type); + startCount++; + } + + @Override + protected void stopNativeSensor(int type) { + started.remove(Integer.valueOf(type)); + stopCount++; + } + + @Override + protected boolean readNativeSensor(int type, float[] out) { + // The background poll loop never produces readings; tests push + // events explicitly through fire(). + return false; + } + + /** + * Pushes a reading into the sensor of the given type, exactly as the + * platform poll loop would. + */ + public void fire(int type, float x, float y, float z, long timestampMillis) { + getSensor(type).dispatch(x, y, z, timestampMillis); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index 4e9716a4c10..168308c876c 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -895,6 +895,41 @@ public com.codename1.impl.CameraImpl createCameraImpl() { return cameraImpl; } + private com.codename1.impl.ARImpl arImpl; + + /** + * Installs the {@link com.codename1.impl.ARImpl} backend that + * {@link com.codename1.ar.AR} sees through {@code Display.getARBackend()}. + * Pass {@code null} (the default) to model a platform with no AR support. + */ + public void setARImpl(com.codename1.impl.ARImpl arImpl) { + this.arImpl = arImpl; + } + + @Override + public com.codename1.impl.ARImpl createARImpl() { + return arImpl; + } + + private com.codename1.sensors.MotionSensorManager motionSensorManager; + + /** + * Installs the {@link com.codename1.sensors.MotionSensorManager} returned + * by {@code Display.getMotionSensorManager()}. Pass {@code null} (the + * default) to fall back to the base implementation's unsupported manager. + */ + public void setMotionSensorManager(com.codename1.sensors.MotionSensorManager mgr) { + this.motionSensorManager = mgr; + } + + @Override + public com.codename1.sensors.MotionSensorManager getMotionSensorManager() { + if (motionSensorManager != null) { + return motionSensorManager; + } + return super.getMotionSensorManager(); + } + @Override public PeerComponent createBrowserComponent(Object browserComponent) { if(this.browserComponent == null) { @@ -1235,6 +1270,8 @@ public void reset() { largerTextEnabled = false; largerTextScale = 1f; cameraImpl = null; + arImpl = null; + motionSensorManager = null; platformName = "test"; } diff --git a/maven/core-unittests/src/test/java/com/codename1/vr/HeadTrackerTest.java b/maven/core-unittests/src/test/java/com/codename1/vr/HeadTrackerTest.java new file mode 100644 index 00000000000..31562ae9972 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/vr/HeadTrackerTest.java @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.junit.UITestBase; +import com.codename1.sensors.FakeMotionSensorManager; +import com.codename1.sensors.MotionSensorManager; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for {@link HeadTracker}: sensor lifecycle wiring through + * {@link FakeMotionSensorManager}, gyro-driven orientation updates, the + * no-gyro accelerometer fallback and thread-safe snapshot reads. + */ +class HeadTrackerTest extends UITestBase { + + private static float yawAngle(float[] q) { + return 2f * (float) Math.atan2(q[1], q[3]); + } + + @Test + void isSupportedReflectsInstalledSensors() { + implementation.setMotionSensorManager(new FakeMotionSensorManager( + MotionSensorManager.TYPE_GYROSCOPE)); + assertTrue(HeadTracker.isSupported()); + + implementation.setMotionSensorManager(new FakeMotionSensorManager( + MotionSensorManager.TYPE_ACCELEROMETER)); + assertTrue(HeadTracker.isSupported()); + + implementation.setMotionSensorManager(new FakeMotionSensorManager()); + assertFalse(HeadTracker.isSupported()); + } + + @Test + void eventsOnlyDriveTheOrientationWhileStarted() { + // Native sensor start/stop happens on the manager's background poll + // thread, so this test observes the listener wiring instead: events + // fired before start() or after stop() must not move the orientation. + FakeMotionSensorManager mgr = new FakeMotionSensorManager( + MotionSensorManager.TYPE_GYROSCOPE, + MotionSensorManager.TYPE_ACCELEROMETER, + MotionSensorManager.TYPE_MAGNETOMETER); + HeadTracker tracker = new HeadTracker(mgr); + assertFalse(tracker.isStarted()); + + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 1000); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 1100); + flushSerialCalls(); + float[] q = new float[4]; + tracker.getOrientation(q); + assertEquals(0f, yawAngle(q), 0f, "events before start are ignored"); + + tracker.start(); + tracker.start(); + assertTrue(tracker.isStarted()); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 2000); + flushSerialCalls(); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 2100); + flushSerialCalls(); + tracker.getOrientation(q); + float yawWhileRunning = yawAngle(q); + assertTrue(yawWhileRunning > 0.05f, "events while started move the orientation"); + // The double start did not register duplicate listeners: one 100ms + // step at 1 rad/s is 0.1 radians, not 0.2. + assertEquals(0.1f, yawWhileRunning, 0.01f); + + tracker.stop(); + tracker.stop(); + assertFalse(tracker.isStarted()); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 3000); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 3100); + flushSerialCalls(); + tracker.getOrientation(q); + assertEquals(yawWhileRunning, yawAngle(q), 1e-6f, "events after stop are ignored"); + } + + @Test + void gyroEventsUpdateTheOrientation() { + FakeMotionSensorManager mgr = new FakeMotionSensorManager( + MotionSensorManager.TYPE_GYROSCOPE); + HeadTracker tracker = new HeadTracker(mgr); + tracker.start(); + + // 1 rad/s around device Y; first event only establishes the clock. + long t = 1000; + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, t); + flushSerialCalls(); + for (int i = 0; i < 50; i++) { + t += 10; + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, t); + flushSerialCalls(); + } + float[] q = new float[4]; + tracker.getOrientation(q); + // 50 steps of 10ms at 1 rad/s = 0.5 radians of yaw. + assertEquals(0.5f, yawAngle(q), 0.01f); + tracker.stop(); + } + + @Test + void accelerometerFallbackConvergesTiltWithoutGyro() { + FakeMotionSensorManager mgr = new FakeMotionSensorManager( + MotionSensorManager.TYPE_ACCELEROMETER); + HeadTracker tracker = new HeadTracker(mgr); + tracker.getFilter().setGyroWeight(0.5f); + tracker.start(); + + // Device pitched so gravity reaction reads along device +Z. + long t = 1000; + for (int i = 0; i < 200; i++) { + mgr.fire(MotionSensorManager.TYPE_ACCELEROMETER, 0f, 0f, 9.81f, t); + flushSerialCalls(); + t += 10; + } + float[] q = new float[4]; + tracker.getOrientation(q); + float[] deviceUp = {0f, 0f, 1f}; + com.codename1.gpu.Quaternion.rotateVector(q, deviceUp); + assertEquals(1f, deviceUp[1], 0.01f, "device +Z should map to world up"); + tracker.stop(); + } + + @Test + void recenterZeroesTheYaw() { + FakeMotionSensorManager mgr = new FakeMotionSensorManager( + MotionSensorManager.TYPE_GYROSCOPE); + HeadTracker tracker = new HeadTracker(mgr); + tracker.start(); + long t = 1000; + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 2f, 0f, t); + flushSerialCalls(); + for (int i = 0; i < 30; i++) { + t += 10; + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 2f, 0f, t); + flushSerialCalls(); + } + float[] q = new float[4]; + tracker.getOrientation(q); + assertTrue(Math.abs(yawAngle(q)) > 0.1f); + tracker.recenter(); + tracker.getOrientation(q); + assertEquals(0f, yawAngle(q), 0.01f); + tracker.stop(); + } + + @Test + void snapshotIsReadableFromAnotherThread() throws InterruptedException { + FakeMotionSensorManager mgr = new FakeMotionSensorManager( + MotionSensorManager.TYPE_GYROSCOPE); + final HeadTracker tracker = new HeadTracker(mgr); + tracker.start(); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 1000); + flushSerialCalls(); + mgr.fire(MotionSensorManager.TYPE_GYROSCOPE, 0f, 1f, 0f, 1100); + flushSerialCalls(); + + final AtomicReference fromThread = new AtomicReference(); + Thread render = new Thread(new Runnable() { + public void run() { + float[] q = new float[4]; + tracker.getOrientation(q); + fromThread.set(q); + } + }, "fake-render-thread"); + render.start(); + render.join(5000); + assertNotNull(fromThread.get()); + float len = (float) Math.sqrt(fromThread.get()[0] * fromThread.get()[0] + + fromThread.get()[1] * fromThread.get()[1] + + fromThread.get()[2] * fromThread.get()[2] + + fromThread.get()[3] * fromThread.get()[3]); + assertEquals(1f, len, 1e-4f); + tracker.stop(); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/vr/Media360ViewTest.java b/maven/core-unittests/src/test/java/com/codename1/vr/Media360ViewTest.java new file mode 100644 index 00000000000..57ca06c7d5b --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/vr/Media360ViewTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.junit.UITestBase; +import com.codename1.ui.Image; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for {@link Media360View} on the headless test platform: without a + * GPU backend the component must degrade gracefully, and its look-angle state + * machine (yaw wrap, pitch clamp, drag deltas, reset) must behave. + */ +class Media360ViewTest extends UITestBase { + + @Test + void constructsWithoutGpuBackendAndReportsUnsupported() { + Media360View view = new Media360View(); + assertFalse(view.isSupported()); + assertNotNull(view.getRenderView()); + // setImage before any GPU init must not crash headless. + view.setImage(Image.createImage(4, 2, 0xff112233)); + } + + @Test + void yawAndPitchSettersWithPitchClamp() { + Media360View view = new Media360View(); + view.setYaw(123f); + assertEquals(123f, view.getYaw(), 0f); + view.setPitch(45f); + assertEquals(45f, view.getPitch(), 0f); + view.setPitch(120f); + assertEquals(89f, view.getPitch(), 0f, "pitch clamps at the pole"); + view.setPitch(-120f); + assertEquals(-89f, view.getPitch(), 0f); + } + + @Test + void dragUpdatesYawAndPitch() { + Media360View view = new Media360View(); + view.setWidth(360); + view.setHeight(180); + view.pointerPressed(100, 100); + // Drag right by 36px on a 360px wide view = 18 degrees; the view + // turns the opposite way (grab-the-world). + view.pointerDragged(136, 100); + assertEquals(-18f, view.getYaw(), 0.01f); + // Drag down by 18px on a 180px tall view = 18 degrees pitch up. + view.pointerDragged(136, 118); + assertEquals(18f, view.getPitch(), 0.01f); + view.pointerReleased(136, 118); + + // A fresh press does not jump: deltas restart from the new point. + view.pointerPressed(0, 0); + view.pointerDragged(0, 0); + assertEquals(-18f, view.getYaw(), 0.01f); + } + + @Test + void pitchClampsDuringDrag() { + Media360View view = new Media360View(); + view.setWidth(100); + view.setHeight(100); + view.pointerPressed(50, 0); + view.pointerDragged(50, 100); + view.pointerDragged(50, 200); + assertEquals(89f, view.getPitch(), 0f); + } + + @Test + void resetRestoresStraightAhead() { + Media360View view = new Media360View(); + view.setYaw(90f); + view.setPitch(-30f); + view.reset(); + assertEquals(0f, view.getYaw(), 0f); + assertEquals(0f, view.getPitch(), 0f); + } + + @Test + void stereoToggle() { + Media360View view = new Media360View(); + assertFalse(view.isStereo()); + view.setStereo(true); + assertTrue(view.isStereo()); + } + + @Test + void headTrackingToggleWithoutSensorsIsSafe() { + implementation.setMotionSensorManager(new com.codename1.sensors.FakeMotionSensorManager()); + Media360View view = new Media360View(); + view.setHeadTrackingEnabled(true); + assertTrue(view.isHeadTrackingEnabled()); + view.setHeadTrackingEnabled(false); + assertFalse(view.isHeadTrackingEnabled()); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/vr/OrientationFilterTest.java b/maven/core-unittests/src/test/java/com/codename1/vr/OrientationFilterTest.java new file mode 100644 index 00000000000..f8ead2aea6a --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/vr/OrientationFilterTest.java @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Quaternion; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for the deterministic {@link OrientationFilter}: pure math with no + * Display dependency, so identical inputs must always give identical output. + */ +class OrientationFilterTest { + + private static final float EPS = 1e-3f; + + private static float angleAroundAxis(float[] q, float ax, float ay, float az) { + // Rotation angle of q assuming it is (close to) a rotation around the + // given unit axis. + float dot = q[0] * ax + q[1] * ay + q[2] * az; + return 2f * (float) Math.atan2(dot, q[3]); + } + + @Test + void zeroInputKeepsIdentity() { + OrientationFilter f = new OrientationFilter(); + for (int i = 0; i < 100; i++) { + f.update(0, 0, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.01f); + } + assertArrayEquals(new float[]{0, 0, 0, 1}, f.getOrientation(), EPS); + } + + @Test + void pureGyroIntegratesTheExpectedAngle() { + OrientationFilter f = new OrientationFilter(); + // 0.5 rad/s around device Y for 2 seconds in 10ms steps = 1 radian. + for (int i = 0; i < 200; i++) { + f.update(0, 0.5f, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.01f); + } + float[] q = f.getOrientation(); + assertEquals(1f, angleAroundAxis(q, 0, 1, 0), 1e-2f); + } + + @Test + void identicalInputsAreDeterministic() { + OrientationFilter a = new OrientationFilter(); + OrientationFilter b = new OrientationFilter(); + for (int i = 0; i < 500; i++) { + float gx = (i % 7) * 0.01f; + float ay = 9.5f + (i % 3) * 0.1f; + a.update(gx, 0.2f, -0.1f, 0.3f, ay, 0.2f, 20f, 0f, -30f, 0.01f); + b.update(gx, 0.2f, -0.1f, 0.3f, ay, 0.2f, 20f, 0f, -30f, 0.01f); + } + assertArrayEquals(a.getOrientation(), b.getOrientation()); + } + + @Test + void accelerometerConvergesTiltTowardGravity() { + OrientationFilter f = new OrientationFilter(); + f.setGyroWeight(0.9f); + // Device pitched forward 90 degrees: gravity reaction along device +Z + // (the device Z axis points up in the world). The filter should + // converge to an orientation whose world-up maps to device +Z. + for (int i = 0; i < 2000; i++) { + f.update(0, 0, 0, 0, 0, 9.81f, Float.NaN, Float.NaN, Float.NaN, 0.01f); + } + float[] q = f.getOrientation(); + float[] deviceUp = {0, 0, 1}; + Quaternion.rotateVector(q, deviceUp); + assertEquals(0f, deviceUp[0], EPS); + assertEquals(1f, deviceUp[1], EPS); + assertEquals(0f, deviceUp[2], EPS); + } + + @Test + void higherGyroWeightConvergesSlower() { + OrientationFilter fast = new OrientationFilter(); + fast.setGyroWeight(0.5f); + OrientationFilter slow = new OrientationFilter(); + slow.setGyroWeight(0.99f); + for (int i = 0; i < 10; i++) { + fast.update(0, 0, 0, 0, 0, 9.81f, Float.NaN, Float.NaN, Float.NaN, 0.01f); + slow.update(0, 0, 0, 0, 0, 9.81f, Float.NaN, Float.NaN, Float.NaN, 0.01f); + } + float[] up = {0, 0, 1}; + float[] q = fast.getOrientation(); + Quaternion.rotateVector(q, up); + float fastY = up[1]; + up[0] = 0; + up[1] = 0; + up[2] = 1; + q = slow.getOrientation(); + Quaternion.rotateVector(q, up); + float slowY = up[1]; + assertTrue(fastY > slowY, "lower gyro weight must correct tilt faster"); + } + + @Test + void magnetometerCorrectsYawAndNaNSkipsIt() { + // Start yawed 45 degrees off; with north (-Z field in the device + // frame when facing north) the magnetometer should pull yaw back. + OrientationFilter f = new OrientationFilter(); + f.setGyroWeight(0.9f); + // Introduce a yaw error via gyro. + f.update(0, 0.785f, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 1f); + float before = Math.abs(angleAroundAxis(f.getOrientation(), 0, 1, 0)); + // Field pointing along device -Z (device facing north, flat field). + for (int i = 0; i < 500; i++) { + f.update(0, 0, 0, 0, 0, 0, 0f, 0f, -30f, 0.01f); + } + float after = Math.abs(angleAroundAxis(f.getOrientation(), 0, 1, 0)); + assertTrue(after < before * 0.1f, "yaw error should shrink: " + before + " -> " + after); + + // NaN magnetometer leaves yaw untouched. + OrientationFilter g = new OrientationFilter(); + g.update(0, 0.785f, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 1f); + float yawBefore = angleAroundAxis(g.getOrientation(), 0, 1, 0); + for (int i = 0; i < 100; i++) { + g.update(0, 0, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.01f); + } + assertEquals(yawBefore, angleAroundAxis(g.getOrientation(), 0, 1, 0), EPS); + } + + @Test + void dtScalingIsConsistent() { + OrientationFilter one = new OrientationFilter(); + OrientationFilter two = new OrientationFilter(); + // One 20ms step vs two 10ms steps of the same constant rate. + one.update(0.4f, 0, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.02f); + two.update(0.4f, 0, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.01f); + two.update(0.4f, 0, 0, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.01f); + assertArrayEquals(one.getOrientation(), two.getOrientation(), 1e-4f); + } + + @Test + void quaternionStaysNormalizedOverLongRuns() { + OrientationFilter f = new OrientationFilter(); + for (int i = 0; i < 20000; i++) { + f.update(0.3f, -0.2f, 0.15f, 0.5f, 9.7f, 0.4f, 25f, -10f, -35f, 0.005f); + } + float[] q = f.getOrientation(); + float len = (float) Math.sqrt(q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]); + assertEquals(1f, len, 1e-3f); + } + + @Test + void recenterYawZeroesYawAndKeepsPitch() { + OrientationFilter f = new OrientationFilter(); + // Yaw 60 degrees then pitch up 30 degrees (device frame). + f.update(0, (float) Math.toRadians(60), 0, 0, 0, 0, + Float.NaN, Float.NaN, Float.NaN, 1f); + f.update((float) Math.toRadians(30), 0, 0, 0, 0, 0, + Float.NaN, Float.NaN, Float.NaN, 1f); + f.recenterYaw(); + float[] q = f.getOrientation(); + // Forward direction should now have no horizontal deviation. + float[] fwd = {0, 0, -1}; + Quaternion.rotateVector(q, fwd); + assertEquals(0f, fwd[0], EPS); + assertTrue(fwd[2] < 0, "still facing forward"); + // The pitch (vertical component) survives the recenter. + assertEquals(Math.sin(Math.toRadians(30)), fwd[1], 1e-2f); + } + + @Test + void invalidGyroWeightRejected() { + final OrientationFilter f = new OrientationFilter(); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + f.setGyroWeight(0f); + } + }); + assertThrows(IllegalArgumentException.class, new org.junit.jupiter.api.function.Executable() { + public void execute() { + f.setGyroWeight(1.5f); + } + }); + } + + @Test + void resetReturnsToIdentity() { + OrientationFilter f = new OrientationFilter(); + f.update(1f, 2f, 3f, 0, 0, 0, Float.NaN, Float.NaN, Float.NaN, 0.5f); + f.reset(); + assertArrayEquals(new float[]{0, 0, 0, 1}, f.getOrientation()); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/vr/VRCameraRigTest.java b/maven/core-unittests/src/test/java/com/codename1/vr/VRCameraRigTest.java new file mode 100644 index 00000000000..c5b8c82a8bf --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/vr/VRCameraRigTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.vr; + +import com.codename1.gpu.Camera; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Quaternion; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Coverage for the {@link VRCameraRig} per-eye camera math: interpupillary + * separation along the head-right vector, orientation handling and lens + * parameter propagation. Pure math, no Display required. + */ +class VRCameraRigTest { + + private static final float EPS = 1e-5f; + + @Test + void identityOrientationSeparatesEyesAlongX() { + VRCameraRig rig = new VRCameraRig(new VRSettings().ipdMeters(0.064f)); + Camera left = new Camera(); + Camera right = new Camera(); + rig.apply(left, VREye.LEFT, 1f); + rig.apply(right, VREye.RIGHT, 1f); + assertEquals(-0.032f, left.getEyeX(), EPS); + assertEquals(0.032f, right.getEyeX(), EPS); + assertEquals(0f, left.getEyeY(), EPS); + assertEquals(0f, left.getEyeZ(), EPS); + // Total separation is exactly the IPD. + assertEquals(0.064f, right.getEyeX() - left.getEyeX(), EPS); + } + + @Test + void centerEyeHasNoOffset() { + VRCameraRig rig = new VRCameraRig(new VRSettings().ipdMeters(0.064f)); + rig.setPosition(1f, 2f, 3f); + Camera center = new Camera(); + rig.apply(center, VREye.CENTER, 1f); + assertEquals(1f, center.getEyeX(), EPS); + assertEquals(2f, center.getEyeY(), EPS); + assertEquals(3f, center.getEyeZ(), EPS); + } + + @Test + void yawRotationRotatesTheSeparationAxis() { + VRCameraRig rig = new VRCameraRig(new VRSettings().ipdMeters(0.1f)); + // Head yawed 90 degrees left (counterclockwise around Y): the head's + // right vector becomes world -Z. + rig.setOrientation(Quaternion.fromAxisAngle((float) Math.PI / 2, 0, 1, 0)); + Camera left = new Camera(); + Camera right = new Camera(); + rig.apply(left, VREye.LEFT, 1f); + rig.apply(right, VREye.RIGHT, 1f); + assertEquals(0f, left.getEyeX(), EPS); + assertEquals(0.05f, left.getEyeZ(), EPS); + assertEquals(-0.05f, right.getEyeZ(), EPS); + } + + @Test + void viewMatrixMatchesLookAtAlongTheHeadForward() { + VRCameraRig rig = new VRCameraRig(new VRSettings().ipdMeters(0f)); + rig.setPosition(0f, 1.6f, 0f); + Camera cam = new Camera(); + rig.apply(cam, VREye.CENTER, 1f); + // Identity orientation looks along -Z with +Y up. + float[] expected = Matrix4.lookAt(0f, 1.6f, 0f, 0f, 1.6f, -1f, 0f, 1f, 0f); + float[] actual = cam.getViewMatrix(); + for (int i = 0; i < 16; i++) { + assertEquals(expected[i], actual[i], EPS, "element " + i); + } + } + + @Test + void lensParametersPropagateToTheProjection() { + VRCameraRig rig = new VRCameraRig( + new VRSettings().fovYDegrees(100f).nearFar(0.5f, 200f)); + Camera cam = new Camera(); + rig.apply(cam, VREye.CENTER, 2f); + float[] expected = Matrix4.perspective((float) Math.toRadians(100f), 2f, 0.5f, 200f); + float[] actual = cam.getProjectionMatrix(); + for (int i = 0; i < 16; i++) { + assertEquals(expected[i], actual[i], EPS, "element " + i); + } + } + + @Test + void nullSettingsFallBackToDefaults() { + VRCameraRig rig = new VRCameraRig(null); + assertEquals(0.064f, rig.getSettings().getIpdMeters(), EPS); + assertEquals(90f, rig.getSettings().getFovYDegrees(), EPS); + assertTrue(rig.getSettings().isStereo()); + } +} diff --git a/scripts/android/screenshots/Media360Panorama.png b/scripts/android/screenshots/Media360Panorama.png new file mode 100644 index 00000000000..399a4e7c1ad Binary files /dev/null and b/scripts/android/screenshots/Media360Panorama.png differ diff --git a/scripts/android/screenshots/Media360Panorama.tolerance b/scripts/android/screenshots/Media360Panorama.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/android/screenshots/Media360Panorama.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/android/screenshots/VRStereoScene.png b/scripts/android/screenshots/VRStereoScene.png new file mode 100644 index 00000000000..523a4bd8493 Binary files /dev/null and b/scripts/android/screenshots/VRStereoScene.png differ diff --git a/scripts/android/screenshots/VRStereoScene.tolerance b/scripts/android/screenshots/VRStereoScene.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/android/screenshots/VRStereoScene.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/ARApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/ARApiTest.java new file mode 100644 index 00000000000..b582af3f1e2 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/ARApiTest.java @@ -0,0 +1,83 @@ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ar.AR; +import com.codename1.ar.ARCapabilities; +import com.codename1.ar.ARSessionOptions; +import com.codename1.util.SuccessCallback; + +/// Exercises the `com.codename1.ar` API contract against whichever per-port +/// `ARImpl` is in use, without opening a live AR session. +/// +/// A real `AR.open()` has device-specific side effects that make it unsafe in +/// the shared screenshot suite: on Android the ARCore backend calls +/// `ArCoreApk.requestInstall()`, which launches the Play Store install flow and +/// backgrounds the app when ARCore is absent - precisely the CI emulator's +/// state - freezing the whole suite. The full session round trip is therefore +/// covered where it is safe and deterministic: the JavaSE simulator integration +/// test drives `JavaSEARImpl` end to end (open, plane detection, hit test, +/// anchor), and the core unit tests exercise it against `RecordingARImpl`. +/// +/// Here we only verify the parts that are safe on every port: the capability +/// query never returns null, and on platforms with no AR runtime the documented +/// unsupported contract holds (all capabilities false, `AR.open` throws before +/// touching any backend, and the permission request delivers false). +/// +/// No screenshot -- this is an assertion test. +public class ARApiTest extends BaseTest { + + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + final boolean supported = AR.isSupported(); + ARCapabilities caps = AR.getCapabilities(); + if (caps == null) { + fail("getCapabilities() must never return null"); + return true; + } + com.codename1.io.Log.p("ARApiTest: supported=" + supported + + " world=" + caps.isWorldTrackingSupported() + + " planes=" + caps.isPlaneDetectionSupported() + + " images=" + caps.isImageTrackingSupported() + + " faces=" + caps.isFaceTrackingSupported() + + " light=" + caps.isLightEstimationSupported()); + + if (supported) { + // A live session open is exercised by the JavaSE simulator + // integration test and the unit tests, not here, to avoid the + // Android ARCore install flow backgrounding the screenshot suite. + done(); + return true; + } + + // No AR runtime on this platform: the full unsupported contract. When + // unsupported, AR.open throws IllegalStateException before it reaches + // any backend, so there is no device side effect to worry about. + if (caps.isWorldTrackingSupported() || caps.isPlaneDetectionSupported() + || caps.isImageTrackingSupported() || caps.isFaceTrackingSupported() + || caps.isLightEstimationSupported()) { + fail("unsupported platforms must report all capabilities false"); + return true; + } + try { + AR.open(new ARSessionOptions()); + fail("AR.open must throw when unsupported"); + return true; + } catch (IllegalStateException expected) { + // The documented unsupported behavior. + } + AR.requestPermissions(new SuccessCallback() { + public void onSucess(Boolean granted) { + if (Boolean.TRUE.equals(granted)) { + fail("permission request must deliver false when AR is unsupported"); + } else { + done(); + } + } + }); + return true; + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java index 61da8186e41..da8b6595bd8 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java @@ -299,7 +299,7 @@ private static int testTimeoutMs(BaseTest testClass) { new Gpu3DTexturedCubeScreenshotTest(), new Gpu3DModelScreenshotTest(), new Gpu3DAnimationTest(), - // Keep this as the last screenshot test; orientation changes can leak into subsequent screenshots. + // Keep this as the last portrait screenshot test; orientation changes can leak into subsequent screenshots. new OrientationLockScreenshotTest(), new InPlaceEditViewTest(), new BytecodeTranslatorRegressionTest(), @@ -311,6 +311,10 @@ private static int testTimeoutMs(BaseTest testClass) { // prompts). Self-skips on iOS / Android / JS where the open // call would surface an OS dialog. new CameraApiTest(), + // Exercises com.codename1.ar end-to-end: the unsupported + // contract on the CI platforms (none has an AR runtime) and a + // full session round trip when a backend is present. + new ARApiTest(), new SimdLargeAllocaTest(), new StreamApiTest(), new StringApiTest(), @@ -350,6 +354,20 @@ private static int testTimeoutMs(BaseTest testClass) { // SKIPs (no screenshot) where the platform cannot encode (iOS simulator, // unsupported targets). Pixel verification lives in VideoIORoundTripTest. new VideoIODecodedFramesScreenshotTest(), + // VR / 360 immersive views, captured in landscape (where a stereo + // scene / panorama reads naturally) via LandscapeCapture, which + // locks landscape on phones and no-ops on desktop/browser/tv. + // Placed near the end - after DesktopMode and the video screenshot, + // but before the media round-trip test - so nothing GPU-heavy or + // orientation-changing precedes those sensitive tests: on the iOS + // Metal backends DesktopMode's screenshot otherwise grabbed the + // lingering 3D form under the late-present race, and the video + // screenshot is GPU-neighbor sensitive. Nothing captures the screen + // after these, so their landscape state cannot leak. VRStereoScene + // self-skips on tvOS (stereo has no use without a headset); + // Media360Panorama still runs there. + new VRStereoSceneScreenshotTest(), + new Media360PanoramaScreenshotTest(), // VideoIO cross-platform coverage: encodes a 6-frame counting clip with audio, // decodes the frames back and verifies the count order + PCM levels. Assertion // test (no screenshot); SKIPs where the platform can't encode. Deliberately LAST: diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LandscapeCapture.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LandscapeCapture.java new file mode 100644 index 00000000000..8ab110b29da --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LandscapeCapture.java @@ -0,0 +1,85 @@ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ui.CN; +import com.codename1.ui.Form; +import com.codename1.ui.util.UITimer; + +/// Landscape helper for the immersive VR / 360 screenshot tests, which read +/// best in a wide frame. On a phone that can rotate this locks the device to +/// landscape and waits (via {@link #awaitLandscape}) for the rotation to +/// actually land before the capture. On platforms that cannot force +/// orientation - desktop, browser, tvOS - it is a no-op and the fixed window +/// aspect is captured as-is. +/// +/// The last of the VR / 360 tests calls {@link #restorePortrait} once its +/// capture is done, so the rest of the suite runs in portrait exactly as it +/// does on master (in particular the video tests, which are sensitive to both +/// orientation and to a GPU-heavy neighbor - hence these tests sit right after +/// the orientation test, far from them). +final class LandscapeCapture { + private static final int POLL_INTERVAL_MS = 100; + // ~4s is comfortably longer than an emulator rotate + layout pass while + // staying well under the per-test timeout; platforms that never rotate + // short-circuit below and never wait this long. + private static final int POLL_ATTEMPTS = 40; + + private LandscapeCapture() { + } + + /// Locks the device to landscape where the platform supports it. Safe to + /// call before the form is shown. + static void lock() { + if (CN.canForceOrientation()) { + CN.lockOrientation(false); // false == landscape + } + } + + /// Invokes {@code onReady} once {@code form} is in landscape, or + /// immediately on platforms that cannot rotate (or are already landscape). + static void awaitLandscape(Form form, Runnable onReady) { + awaitLandscape(form, POLL_ATTEMPTS, onReady); + } + + /// Restores portrait (where the platform can rotate) and invokes + /// {@code onDone} once the rotation has landed, so the next test starts in + /// the same orientation the rest of the suite expects. A no-op that runs + /// {@code onDone} immediately on platforms that cannot rotate. + static void restorePortrait(Form form, Runnable onDone) { + if (!CN.canForceOrientation()) { + onDone.run(); + return; + } + CN.lockOrientation(true); // true == portrait + awaitPortrait(form, POLL_ATTEMPTS, onDone); + } + + private static void awaitPortrait(final Form form, final int attemptsLeft, final Runnable onDone) { + form.revalidate(); + boolean portrait = form.getHeight() >= form.getWidth(); + if (portrait || attemptsLeft <= 0) { + onDone.run(); + return; + } + UITimer.timer(POLL_INTERVAL_MS, false, form, new Runnable() { + public void run() { + awaitPortrait(form, attemptsLeft - 1, onDone); + } + }); + } + + private static void awaitLandscape(final Form form, final int attemptsLeft, final Runnable onReady) { + form.revalidate(); + boolean landscape = form.getWidth() > form.getHeight(); + // Give up waiting on platforms that can't rotate (already at their + // fixed aspect) so they capture immediately with no penalty. + if (landscape || !CN.canForceOrientation() || attemptsLeft <= 0) { + onReady.run(); + return; + } + UITimer.timer(POLL_INTERVAL_MS, false, form, new Runnable() { + public void run() { + awaitLandscape(form, attemptsLeft - 1, onReady); + } + }); + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Media360PanoramaScreenshotTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Media360PanoramaScreenshotTest.java new file mode 100644 index 00000000000..e6e0e607001 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Media360PanoramaScreenshotTest.java @@ -0,0 +1,157 @@ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ui.Form; +import com.codename1.ui.Image; +import com.codename1.ui.layouts.BorderLayout; +import com.codename1.ui.util.UITimer; +import com.codename1.vr.Media360View; + +/// End-to-end screenshot test for the 360 panorama viewer +/// (com.codename1.vr.Media360View). A synthetic equirectangular panorama - +/// sky and ground bands, a sun disc and meridian stripes - is generated in +/// code so no asset is needed, then shown in side-by-side stereo at a fixed +/// look angle with head tracking off, making the capture deterministic. The +/// view is captured in landscape, where a panorama reads naturally. The +/// vertical color transitions also verify the sphere's u direction: the +/// stripes must appear in their generated order, not mirrored. On platforms +/// without a 3D backend the screenshot is skipped; unlike the stereo VR test +/// this one still runs on tvOS, where panning a 360 image is a sensible use. +public class Media360PanoramaScreenshotTest extends BaseTest { + private Media360View view; + + @Override + public boolean runTest() { + // Uses createForm (like VRStereoScene) rather than a hand-rolled Form: + // the immersive tests run at the suite tail, so nothing needs the peer + // detached or portrait restored afterwards, and the plain-Form capture + // path was what left the iOS Metal simulator reading a stale frame here + // while the identical createForm path worked for VRStereoScene. + Form form = createForm("360 Panorama", new BorderLayout(), "Media360Panorama"); + view = new Media360View(); + if (!view.isSupported()) { + done(); + return true; + } + view.setImage(generatePanorama()); + view.setStereo(true); + // Sensors differ per platform (the CI Android emulator reports a + // live rotation vector); freeze head tracking so the look angle is + // exactly the yaw/pitch set below. + view.setHeadTrackingEnabled(false); + // A fixed off-axis look angle exercises the sphere mapping more than + // the straight-ahead default. + view.setYaw(30f); + view.setPitch(10f); + LandscapeCapture.lock(); + form.add(BorderLayout.CENTER, view); + form.show(); + return true; + } + + /// Sky gradient over ground, a sun disc and four meridian stripes of + /// distinct colors, composed directly into an ARGB pixel array. The result + /// is an immutable, array-backed image (Image.createImage(int[], w, h)): + /// texturing it only reads the backing array, never a surface. A mutable + /// image built with a Graphics would instead force the GPU device to read + /// its pixels back (createTexture calls Image.getRGB), which on the iOS + /// Metal simulator returns white / drops the first present so the panorama + /// never appeared. Building the pixels by hand also keeps the texture + /// identical on every platform (no fillArc anti-aliasing variance). + private static Image generatePanorama() { + int w = 1024; + int h = 512; + int[] px = new int[w * h]; + // Sky gradient over the top half. + for (int y = 0; y < h / 2; y++) { + int t = 255 * y / (h / 2); + int color = 0xff000000 | (0x30 << 16) + | ((0x60 + t / 3) << 8) | Math.min(255, 0xa0 + t / 2); + int row = y * w; + for (int x = 0; x < w; x++) { + px[row + x] = color; + } + } + // Ground gradient over the bottom half. + for (int y = h / 2; y < h; y++) { + int t = 255 * (y - h / 2) / (h / 2); + int color = 0xff000000 | ((0x60 - t / 8) << 16) + | ((0x50 - t / 8) << 8) | 0x30; + int row = y * w; + for (int x = 0; x < w; x++) { + px[row + x] = color; + } + } + // Sun disc centered at (w/4, h/4), radius 40, hard-edged. + int cx = w / 4; + int cy = h / 4; + int r2 = 40 * 40; + for (int y = cy - 40; y <= cy + 40; y++) { + if (y < 0 || y >= h) { + continue; + } + int dy = y - cy; + int row = y * w; + for (int x = cx - 40; x <= cx + 40; x++) { + if (x < 0 || x >= w) { + continue; + } + int dx = x - cx; + if (dx * dx + dy * dy <= r2) { + px[row + x] = 0xffffeeaa; + } + } + } + // Four meridian stripes, 8px wide and h/3 tall from y=h/3. + int[] stripeColors = {0xffffffff, 0xffdd4433, 0xff33aa55, 0xff3366ff}; + for (int i = 0; i < 4; i++) { + int sx = (i * w / 4 + w / 2) % w; + int color = stripeColors[i]; + for (int y = h / 3; y < h / 3 + h / 3; y++) { + int row = y * w; + for (int x = sx - 4; x < sx + 4; x++) { + if (x < 0 || x >= w) { + continue; + } + px[row + x] = color; + } + } + } + return Image.createImage(px, w, h); + } + + @Override + public boolean shouldTakeScreenshot() { + return com.codename1.ui.CN.isGpuSupported(); + } + + /// Force a fresh, fully-presented frame before the capture. On the iOS + /// Metal backend a screenshot can otherwise read a previous form's still + /// current drawable (the late-present race); DesktopMode uses the same + /// mitigation. A no-op cost on the other backends. + @Override + protected long extraSettleBeforeCaptureMillis() { + return 700; + } + + /// Wait for landscape to settle, warm the panorama texture with an early + /// render, then force a fresh GPU frame before the capture fires - mirroring + /// VRStereoScene but with extra time for the 1024x512 texture upload. + @Override + protected void registerReadyCallback(final Form parent, final Runnable run) { + LandscapeCapture.awaitLandscape(parent, new Runnable() { + public void run() { + if (view != null) { + view.getRenderView().requestRender(); + } + UITimer.timer(1500, false, parent, new Runnable() { + public void run() { + if (view != null) { + view.getRenderView().requestRender(); + } + UITimer.timer(700, false, parent, run); + } + }); + } + }); + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VRStereoSceneScreenshotTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VRStereoSceneScreenshotTest.java new file mode 100644 index 00000000000..1e63defae8b --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VRStereoSceneScreenshotTest.java @@ -0,0 +1,123 @@ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.gpu.GraphicsDevice; +import com.codename1.gpu.Light; +import com.codename1.gpu.Material; +import com.codename1.gpu.Matrix4; +import com.codename1.gpu.Mesh; +import com.codename1.gpu.Primitives; +import com.codename1.gpu.Camera; +import com.codename1.ui.Form; +import com.codename1.ui.layouts.BorderLayout; +import com.codename1.ui.util.UITimer; +import com.codename1.vr.VREye; +import com.codename1.vr.VRRenderer; +import com.codename1.vr.VRView; + +/// End-to-end screenshot test for the stereoscopic VR API (com.codename1.vr). +/// A {@link VRView} renders a fixed scene - three Phong-lit cubes over a grey +/// floor - in side-by-side stereo. Head tracking is disabled so the capture is +/// deterministic on every platform regardless of sensor availability; the +/// stereo parallax between the two eye views is the feature under test. The +/// view is captured in landscape, where a stereo scene reads naturally. On +/// platforms without a 3D backend the view shows its placeholder and the +/// screenshot is skipped, and on tvOS side-by-side stereo has no use (no +/// headset), so the test is skipped there. +public class VRStereoSceneScreenshotTest extends BaseTest { + private VRView view; + + @Override + public boolean runTest() { + if (com.codename1.ui.CN.isTV()) { + // Stereo VR targets a headset/cardboard viewer; on a TV it is + // redundant, so skip rather than baseline a nonsensical capture. + System.out.println("CN1SS:INFO:test=VRStereoScene status=SKIPPED reason=tv-no-stereo-vr"); + done(); + return true; + } + Form form = createForm("VR Stereo", new BorderLayout(), "VRStereoScene"); + view = new VRView(new VRRenderer() { + private Mesh cube; + private Mesh floor; + private Material blue; + private Material red; + private Material green; + private Material grey; + + public void onInit(GraphicsDevice device) { + cube = Primitives.cube(device, 0.5f); + floor = Primitives.quad(device, 8f); + blue = new Material(Material.Type.PHONG).setColor(0xff3366ff); + red = new Material(Material.Type.PHONG).setColor(0xffdd4433); + green = new Material(Material.Type.PHONG).setColor(0xff33aa55); + grey = new Material(Material.Type.PHONG).setColor(0xff555560); + device.setLight(new Light().setDirection(-0.4f, -1f, -0.55f)); + } + + public void onEyeFrame(GraphicsDevice device, VREye eye, Camera camera) { + // Floor 1.4m below the viewer, laid flat. + float[] floorM = new float[16]; + Matrix4.multiply(Matrix4.translation(0f, -1.4f, -2f), + Matrix4.rotation((float) (-Math.PI / 2), 1f, 0f, 0f), floorM); + device.draw(floor, grey, floorM); + // A fixed-orientation cube ahead and two offset cubes for + // depth cues; near geometry shows the largest eye parallax. + float[] center = new float[16]; + Matrix4.multiply(Matrix4.translation(0f, 0f, -2f), + Matrix4.rotation((float) Math.toRadians(30), 0.4f, 1f, 0.2f), center); + device.draw(cube, blue, center); + device.draw(cube, red, Matrix4.translation(-1.2f, -0.3f, -2.6f)); + device.draw(cube, green, Matrix4.translation(1.2f, 0.3f, -3.2f)); + } + + public void onDispose(GraphicsDevice device) { + } + }); + // Sensors differ per platform (and are absent on CI); freeze the head + // orientation so both eye cameras depend only on the VRSettings. + view.setHeadTrackingEnabled(false); + if (!view.isSupported()) { + // No GPU 3D backend on this platform; skip the screenshot rather + // than gate a placeholder that has no per-platform baseline. + done(); + return true; + } + LandscapeCapture.lock(); + form.add(BorderLayout.CENTER, view); + form.show(); + return true; + } + + @Override + public boolean shouldTakeScreenshot() { + return com.codename1.ui.CN.isGpuSupported(); + } + + /// Force a fresh, fully-presented frame before the capture. On the iOS + /// Metal backend a screenshot can otherwise read a previous form's still + /// current drawable (the late-present race); DesktopMode uses the same + /// mitigation. A no-op cost on the other backends. + @Override + protected long extraSettleBeforeCaptureMillis() { + return 700; + } + + /// Wait for landscape to settle, then force a fresh GPU frame to be + /// rendered (and read back for capture) before the screenshot fires, + /// mirroring the Gpu3D screenshot tests. + @Override + protected void registerReadyCallback(final Form parent, final Runnable run) { + LandscapeCapture.awaitLandscape(parent, new Runnable() { + public void run() { + UITimer.timer(1000, false, parent, new Runnable() { + public void run() { + if (view != null) { + view.requestRender(); + } + UITimer.timer(500, false, parent, run); + } + }); + } + }); + } +} diff --git a/scripts/hellocodenameone/common/src/main/kotlin/com/codenameone/examples/hellocodenameone/HelloCodenameOne.kt b/scripts/hellocodenameone/common/src/main/kotlin/com/codenameone/examples/hellocodenameone/HelloCodenameOne.kt index fb480d6ce8f..73c7515637b 100644 --- a/scripts/hellocodenameone/common/src/main/kotlin/com/codenameone/examples/hellocodenameone/HelloCodenameOne.kt +++ b/scripts/hellocodenameone/common/src/main/kotlin/com/codenameone/examples/hellocodenameone/HelloCodenameOne.kt @@ -1,5 +1,6 @@ package com.codenameone.examples.hellocodenameone +import com.codename1.ar.AR import com.codename1.camera.Camera import com.codename1.car.Car import com.codename1.car.CarApplication @@ -37,6 +38,22 @@ open class HelloCodenameOne : Lifecycle() { } catch (t: Throwable) { System.out.println("CN1SS:CAMERA_DIAG:EXCEPTION " + t.javaClass.name + ": " + t.message) } + // Reference the AR API (com.codename1.ar.*) so the build's bytecode + // scanner flips IPhoneBuilder.usesCn1Ar / AndroidGradleBuilder.arSupport: + // this compiles the CN1AR ARKit natives on iOS (INCLUDE_CN1_AR + + // ARKit/SceneKit linking) and keeps the ARCore-backed impl sources + + // gradle dependency on Android. isSupported()/getCapabilities() never + // open a session, so no camera permission prompt is triggered (and both + // report unsupported on simulators without AR hardware). + try { + val arSupported = AR.isSupported() + val arCaps = AR.getCapabilities() + System.out.println("CN1SS:AR_DIAG supported=" + arSupported + + " world=" + arCaps.isWorldTrackingSupported + + " face=" + arCaps.isFaceTrackingSupported) + } catch (t: Throwable) { + System.out.println("CN1SS:AR_DIAG:EXCEPTION " + t.javaClass.name + ": " + t.message) + } // Reference the in-car API (com.codename1.car.*) so the build's bytecode scanner flips // usesCar: this compiles the CarPlay natives on iOS/Mac (CN1_USE_CARPLAY + // CarPlay.framework + the CarPlay scene/entitlement) and injects the Android Auto diff --git a/scripts/ios/screenshots-metal/Media360Panorama.png b/scripts/ios/screenshots-metal/Media360Panorama.png new file mode 100644 index 00000000000..b4b1a2762f8 Binary files /dev/null and b/scripts/ios/screenshots-metal/Media360Panorama.png differ diff --git a/scripts/ios/screenshots-metal/Media360Panorama.tolerance b/scripts/ios/screenshots-metal/Media360Panorama.tolerance new file mode 100644 index 00000000000..3ab147657c0 --- /dev/null +++ b/scripts/ios/screenshots-metal/Media360Panorama.tolerance @@ -0,0 +1,7 @@ +# The iOS Metal backends (iOS Metal + tvOS) render the smooth 360 sky/ground +# gradient with more per-pixel variance run-to-run than the default delta=8 +# GPU tolerance (measured: ~20% of pixels shift by 8-16, <0.2% beyond 16). +# Widen the per-pixel threshold so that gradient noise does not fail the build; +# a real regression still trips the mismatch percent. +maxChannelDelta=28 +maxMismatchPercent=5.0 diff --git a/scripts/ios/screenshots-metal/VRStereoScene.png b/scripts/ios/screenshots-metal/VRStereoScene.png new file mode 100644 index 00000000000..1b999eec262 Binary files /dev/null and b/scripts/ios/screenshots-metal/VRStereoScene.png differ diff --git a/scripts/ios/screenshots-metal/VRStereoScene.tolerance b/scripts/ios/screenshots-metal/VRStereoScene.tolerance new file mode 100644 index 00000000000..67cfcc6747c --- /dev/null +++ b/scripts/ios/screenshots-metal/VRStereoScene.tolerance @@ -0,0 +1,6 @@ +# The iOS Metal backend renders the scene (floor gradient + cubes) with more +# per-pixel variance run-to-run than the default delta=8 GPU tolerance. +# Widen the per-pixel threshold so gradient noise does not fail the build +# while a real regression still trips the mismatch percent. +maxChannelDelta=28 +maxMismatchPercent=5.0 diff --git a/scripts/ios/screenshots-tv/Media360Panorama.png b/scripts/ios/screenshots-tv/Media360Panorama.png new file mode 100644 index 00000000000..54d3b1fe855 Binary files /dev/null and b/scripts/ios/screenshots-tv/Media360Panorama.png differ diff --git a/scripts/ios/screenshots-tv/Media360Panorama.tolerance b/scripts/ios/screenshots-tv/Media360Panorama.tolerance new file mode 100644 index 00000000000..3ab147657c0 --- /dev/null +++ b/scripts/ios/screenshots-tv/Media360Panorama.tolerance @@ -0,0 +1,7 @@ +# The iOS Metal backends (iOS Metal + tvOS) render the smooth 360 sky/ground +# gradient with more per-pixel variance run-to-run than the default delta=8 +# GPU tolerance (measured: ~20% of pixels shift by 8-16, <0.2% beyond 16). +# Widen the per-pixel threshold so that gradient noise does not fail the build; +# a real regression still trips the mismatch percent. +maxChannelDelta=28 +maxMismatchPercent=5.0 diff --git a/scripts/javascript/screenshots/Media360Panorama.png b/scripts/javascript/screenshots/Media360Panorama.png new file mode 100644 index 00000000000..781d2ef1691 Binary files /dev/null and b/scripts/javascript/screenshots/Media360Panorama.png differ diff --git a/scripts/javascript/screenshots/Media360Panorama.tolerance b/scripts/javascript/screenshots/Media360Panorama.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/javascript/screenshots/Media360Panorama.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/javascript/screenshots/VRStereoScene.png b/scripts/javascript/screenshots/VRStereoScene.png new file mode 100644 index 00000000000..2e22a162f1a Binary files /dev/null and b/scripts/javascript/screenshots/VRStereoScene.png differ diff --git a/scripts/javascript/screenshots/VRStereoScene.tolerance b/scripts/javascript/screenshots/VRStereoScene.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/javascript/screenshots/VRStereoScene.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/javase/lib/SimulatorModeTestApp.java b/scripts/javase/lib/SimulatorModeTestApp.java index 5a691bc7b3a..2b65b7a877b 100644 --- a/scripts/javase/lib/SimulatorModeTestApp.java +++ b/scripts/javase/lib/SimulatorModeTestApp.java @@ -1,5 +1,15 @@ package com.codenameone.examples.javase.tests; +import com.codename1.ar.AR; +import com.codename1.ar.ARAnchor; +import com.codename1.ar.ARModel; +import com.codename1.ar.ARNode; +import com.codename1.ar.ARPlane; +import com.codename1.ar.ARPlaneDetection; +import com.codename1.ar.ARPlaneEvent; +import com.codename1.ar.ARSession; +import com.codename1.ar.ARSessionOptions; +import com.codename1.gpu.Primitives; import com.codename1.impl.javase.JavaSEPort; import com.codename1.ui.Button; import com.codename1.ui.CN; @@ -58,6 +68,11 @@ public void start() { current.show(); return; } + if (Boolean.getBoolean("cn1.test.arDemo")) { + current = createArDemoForm(); + current.show(); + return; + } String mode = System.getProperty("cn1.test.window.mode", "unknown"); Form form = new Form("JavaSE Simulator Test", new BorderLayout()); // The Toolbar carries most of the visual identity of a native @@ -138,6 +153,76 @@ public void start() { }); } + /** + * Exercises the simulated AR backend ({@code JavaSEARImpl}) end to end + * the way an application would: open a world-tracking session with + * plane detection, wait for the virtual room's floor plane to be + * "detected", hit-test a fixed screen point against it and anchor a + * red sphere at the hit. Every step is deterministic - the simulated + * camera starts at the origin and the planes appear on fixed delays - + * so the resulting screenshot doubles as a rendering baseline for the + * simulator's AR view. The result line is written to stdout and to the + * sentinel configured by {@code cn1.test.arResultFile} so the outer + * verifier can assert the full pipeline (open, plane events on the EDT, + * hit test, anchor, node attachment) actually ran. + */ + private Form createArDemoForm() { + Form form = new Form("AR Simulator Test", new BorderLayout()); + final Label status = new Label("Waiting for AR planes..."); + form.add(BorderLayout.SOUTH, status); + if (!AR.isSupported()) { + status.setText("AR unsupported"); + reportArResult("result=FAIL reason=unsupported"); + return form; + } + final ARSession session = AR.open(new ARSessionOptions() + .planeDetection(ARPlaneDetection.HORIZONTAL_AND_VERTICAL) + .lightEstimation(true)); + form.add(BorderLayout.CENTER, session.createView()); + final boolean[] placed = new boolean[1]; + session.addPlaneListener(evt -> { + if (placed[0] || evt.getKind() != ARPlaneEvent.Kind.ADDED + || evt.getPlane().getType() != ARPlane.Type.HORIZONTAL_UP) { + return; + } + placed[0] = true; + // (0.5, 0.95) aims below the view center so the ray strikes the + // floor in front of the virtual room's back wall regardless of + // whether the wall plane has been detected yet. + session.hitTest(0.5f, 0.95f).ready(hits -> { + if (hits.length == 0) { + status.setText("AR FAIL: hit test returned no results"); + reportArResult("result=FAIL reason=no-hits"); + return; + } + ARAnchor anchor = hits[0].createAnchor(); + ARNode node = new ARNode(ARModel.fromMesh( + Primitives.sphere(0.25f, 16, 24, false), 0xffdd4433)); + // Rest the sphere on the plane instead of half-sunk into it. + node.setLocalPosition(0f, 0.25f, 0f); + anchor.setNode(node); + status.setText("AR PASS: model anchored on detected floor plane"); + reportArResult("result=PASS hits=" + hits.length + + " hitType=" + hits[0].getType() + + " plane=" + (hits[0].getPlane() != null ? hits[0].getPlane().getType() : "(none)") + + " anchors=" + session.getAnchors().length); + }); + }); + CN.setTimeout(15000, () -> { + if (!placed[0]) { + status.setText("AR FAIL: no floor plane detected"); + reportArResult("result=FAIL reason=no-plane"); + } + }); + return form; + } + + private void reportArResult(String detail) { + String line = "[ar-test] " + detail; + System.out.println(line); + writeSentinel(System.getProperty("cn1.test.arResultFile"), line); + } + /** * Reports whether the active native theme matches {@code expected}. * @@ -192,24 +277,28 @@ private String reportNativeThemeResult(String expected) { // Optional sentinel file so harnesses that can't tail stdout in // realtime can still read the result. The path is configured by // the verifier; absence is fine. - String sentinel = System.getProperty("cn1.test.nativeThemeResultFile"); - if (sentinel != null && !sentinel.isEmpty()) { - try { - Path p = Paths.get(sentinel); - Path parent = p.getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - Files.write(p, (line + System.lineSeparator()).getBytes(StandardCharsets.UTF_8), - StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } catch (Exception ex) { - ex.printStackTrace(); - } - } + writeSentinel(System.getProperty("cn1.test.nativeThemeResultFile"), line); return "Native theme " + result + ": expected=" + expected + " loaded=" + (resolvedKey != null ? resolvedKey : "(none)"); } + private static void writeSentinel(String sentinel, String line) { + if (sentinel == null || sentinel.isEmpty()) { + return; + } + try { + Path p = Paths.get(sentinel); + Path parent = p.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.write(p, (line + System.lineSeparator()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + public void stop() { current = CN.getCurrentForm(); } diff --git a/scripts/javase/lib/SimulatorWindowModeVerifier.java b/scripts/javase/lib/SimulatorWindowModeVerifier.java index 5b3612dcbaf..63a5473f094 100644 --- a/scripts/javase/lib/SimulatorWindowModeVerifier.java +++ b/scripts/javase/lib/SimulatorWindowModeVerifier.java @@ -40,6 +40,8 @@ public static void main(String[] args) { Path nativeThemeSentinel = parsed.nativeTheme != null ? projectDir.resolve("native-theme-result.txt") : null; + boolean arDemo = "ar-demo".equals(parsed.scenario); + Path arSentinel = arDemo ? projectDir.resolve("ar-result.txt") : null; List cmd = new ArrayList(); String javaExec = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; @@ -67,6 +69,10 @@ public static void main(String[] args) { cmd.add("-Dcn1.test.expectedNativeTheme=" + parsed.nativeTheme); cmd.add("-Dcn1.test.nativeThemeResultFile=" + nativeThemeSentinel.toAbsolutePath()); } + if (arDemo) { + cmd.add("-Dcn1.test.arDemo=true"); + cmd.add("-Dcn1.test.arResultFile=" + arSentinel.toAbsolutePath()); + } if (parsed.skinPath != null && parsed.skinPath.length() > 0) { cmd.add("-Dskin=" + parsed.skinPath); cmd.add("-Ddskin=" + parsed.skinPath); @@ -79,7 +85,7 @@ public static void main(String[] args) { ProcessBuilder pb = new ProcessBuilder(cmd); pb.directory(projectDir.toFile()); pb.redirectErrorStream(true); - if (parsed.nativeTheme != null) { + if (parsed.nativeTheme != null || arDemo) { // Capture output to a log so we can also confirm the // result line on stdout if the sentinel file isn't // written for some reason. Without the redirect the @@ -94,7 +100,33 @@ public static void main(String[] args) { waitForSimulatorWarmup(Duration.ofSeconds("network-monitor".equals(parsed.scenario) ? 12 : 8)); + if (arDemo) { + // The AR scenario reaches its steady state (model anchored on + // the detected floor plane) at its own pace; the sentinel file + // is written exactly then, so wait for it instead of guessing + // with a longer sleep. The post-capture assertion below still + // reports the real failure if the deadline passes. + Instant arDeadline = Instant.now().plusSeconds(30); + while (!Files.exists(arSentinel) && Instant.now().isBefore(arDeadline)) { + Thread.sleep(250); + } + // One extra second so the EDT repaint that follows the anchor + // attachment is on screen before the Robot capture. + Thread.sleep(1000); + } + + // The warmup above is a floor for the scenario to reach its + // steady state, but on slow runners the simulator window may not + // have painted yet, which used to fail the run with a blank/flat + // capture. Keep polling until the desktop shows actual content + // (or a generous deadline passes and validation reports the + // real failure). BufferedImage image = captureDesktop(); + Instant renderDeadline = Instant.now().plusSeconds(30); + while (isBlankOrFlat(image) && Instant.now().isBefore(renderDeadline)) { + Thread.sleep(500); + image = captureDesktop(); + } validateScreenshotContent(image); Path screenshotPath = Path.of(parsed.screenshotPath); @@ -109,6 +141,10 @@ public static void main(String[] args) { if (parsed.nativeTheme != null) { assertNativeThemeApplied(parsed, nativeThemeSentinel, projectDir); } + if (arDemo) { + assertResultLinePass("[ar-test]", arSentinel, projectDir, + "AR simulator demo (open session, detect plane, hit test, anchor model)"); + } exitCode = 0; } catch (Throwable t) { t.printStackTrace(System.err); @@ -144,6 +180,17 @@ private static void validateScreenshotContent(BufferedImage image) { if (image.getWidth() < 120 || image.getHeight() < 120) { throw new AssertionError("Screenshot is unexpectedly small: " + image.getWidth() + "x" + image.getHeight()); } + int samples = sampleColorCount(image); + if (samples < 3) { + throw new AssertionError("Screenshot appears blank/flat (insufficient color variation): " + samples); + } + } + + private static boolean isBlankOrFlat(BufferedImage image) { + return sampleColorCount(image) < 3; + } + + private static int sampleColorCount(BufferedImage image) { Set samples = new HashSet(); int stepX = Math.max(1, image.getWidth() / 24); int stepY = Math.max(1, image.getHeight() / 24); @@ -152,9 +199,7 @@ private static void validateScreenshotContent(BufferedImage image) { samples.add(image.getRGB(x, y)); } } - if (samples.size() < 3) { - throw new AssertionError("Screenshot appears blank/flat (insufficient color variation): " + samples.size()); - } + return samples.size(); } /** @@ -191,6 +236,38 @@ private static void assertNativeThemeApplied(Args args, Path sentinel, Path proj } } + /** + * Asserts a scenario result line (sentinel file preferred, captured + * stdout log as fallback) starts with {@code linePrefix} and reports + * {@code result=PASS}. Mirrors the native-theme assertion for + * scenarios that verify behavior in addition to the screenshot. + */ + private static void assertResultLinePass(String linePrefix, Path sentinel, Path projectDir, + String description) throws Exception { + String line = null; + if (sentinel != null && Files.exists(sentinel)) { + line = new String(Files.readAllBytes(sentinel), StandardCharsets.UTF_8).trim(); + } + if (line == null || line.isEmpty()) { + Path log = projectDir.resolve("simulator-output.log"); + if (Files.exists(log)) { + for (String l : Files.readAllLines(log, StandardCharsets.UTF_8)) { + if (l.startsWith(linePrefix)) { + line = l.trim(); + break; + } + } + } + } + if (line == null || line.isEmpty()) { + throw new AssertionError(description + " produced no result line (sentinel=" + sentinel + ")"); + } + System.out.println("[javase-verifier] assertion: " + line); + if (!line.contains("result=PASS")) { + throw new AssertionError(description + " did not pass: " + line); + } + } + private static Path prepareCodenameOneSettings() throws Exception { Path tempProject = Files.createTempDirectory("cn1-javase-sim-project"); Path settings = tempProject.resolve("codenameone_settings.properties"); diff --git a/scripts/javase/screenshots/README.md b/scripts/javase/screenshots/README.md index df3b4239ad1..20bd4c1db1e 100644 --- a/scripts/javase/screenshots/README.md +++ b/scripts/javase/screenshots/README.md @@ -13,10 +13,18 @@ This directory stores baseline PNG files for JavaSE simulator integration screen - `javase-single-native-theme-ios7.png` - `javase-single-native-theme-android-material.png` - `javase-single-native-theme-android-holo.png` +- `javase-single-ar-demo.png` The CI workflow compares generated simulator screenshots with these files. If screenshots differ (or are missing), CI uploads artifacts and posts a PR comment with visual previews. +The `javase-single-ar-demo.png` baseline exercises the simulated AR +backend (`JavaSEARImpl`): the harness opens a session, waits for the +virtual room's floor plane, hit-tests it and anchors a model, and the +capture shows that model resting on the detected floor. A small +`.tolerance` sidecar accompanies it because the anchored model is +rendered through the software 3D rasterizer. + The `javase-single-native-theme-*` baselines exercise the Simulator's "Native Theme" menu - each one runs with the corresponding `simulatorNativeTheme` preference set, the same preference the menu diff --git a/scripts/javase/screenshots/javase-single-ar-demo.png b/scripts/javase/screenshots/javase-single-ar-demo.png new file mode 100644 index 00000000000..a94aec30f62 Binary files /dev/null and b/scripts/javase/screenshots/javase-single-ar-demo.png differ diff --git a/scripts/javase/screenshots/javase-single-ar-demo.tolerance b/scripts/javase/screenshots/javase-single-ar-demo.tolerance new file mode 100644 index 00000000000..cbda72a3229 --- /dev/null +++ b/scripts/javase/screenshots/javase-single-ar-demo.tolerance @@ -0,0 +1,6 @@ +# The AR view renders a 3D scene (grid room + anchored sphere) through the +# JavaSE software rasterizer; antialiased sphere edges can shift by a pixel +# between JVM builds. Small tolerance so that noise does not fail the build +# while still catching a real rendering or layout regression. +maxChannelDelta=8 +maxMismatchPercent=2.0 diff --git a/scripts/linux/screenshots-arm/Media360Panorama.png b/scripts/linux/screenshots-arm/Media360Panorama.png new file mode 100644 index 00000000000..54f14917724 Binary files /dev/null and b/scripts/linux/screenshots-arm/Media360Panorama.png differ diff --git a/scripts/linux/screenshots-arm/VRStereoScene.png b/scripts/linux/screenshots-arm/VRStereoScene.png new file mode 100644 index 00000000000..0f0b02c0462 Binary files /dev/null and b/scripts/linux/screenshots-arm/VRStereoScene.png differ diff --git a/scripts/linux/screenshots/Media360Panorama.png b/scripts/linux/screenshots/Media360Panorama.png new file mode 100644 index 00000000000..982fff0ebdb Binary files /dev/null and b/scripts/linux/screenshots/Media360Panorama.png differ diff --git a/scripts/linux/screenshots/VRStereoScene.png b/scripts/linux/screenshots/VRStereoScene.png new file mode 100644 index 00000000000..0f0b02c0462 Binary files /dev/null and b/scripts/linux/screenshots/VRStereoScene.png differ diff --git a/scripts/mac-native/screenshots/Media360Panorama.png b/scripts/mac-native/screenshots/Media360Panorama.png new file mode 100644 index 00000000000..baa1cfc9928 Binary files /dev/null and b/scripts/mac-native/screenshots/Media360Panorama.png differ diff --git a/scripts/mac-native/screenshots/Media360Panorama.tolerance b/scripts/mac-native/screenshots/Media360Panorama.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/mac-native/screenshots/Media360Panorama.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/mac-native/screenshots/VRStereoScene.png b/scripts/mac-native/screenshots/VRStereoScene.png new file mode 100644 index 00000000000..684f0d91868 Binary files /dev/null and b/scripts/mac-native/screenshots/VRStereoScene.png differ diff --git a/scripts/mac-native/screenshots/VRStereoScene.tolerance b/scripts/mac-native/screenshots/VRStereoScene.tolerance new file mode 100644 index 00000000000..f0c73fe3101 --- /dev/null +++ b/scripts/mac-native/screenshots/VRStereoScene.tolerance @@ -0,0 +1,4 @@ +# GPU 3D render: software rasterizers (swiftshader/Metal sim) vary ~1% +# between CI runners; widen tolerance so noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/run-android-instrumentation-tests.sh b/scripts/run-android-instrumentation-tests.sh index bfba7a333ac..232a8fdce9d 100755 --- a/scripts/run-android-instrumentation-tests.sh +++ b/scripts/run-android-instrumentation-tests.sh @@ -203,7 +203,17 @@ if ! ( fi END_MARKER="CN1SS:SUITE:FINISHED" -TIMEOUT_SECONDS=60 +# The instrumentation @Test is a trivial launcher; the CN1SS suite runs +# asynchronously in the app process, so this wait is effectively the suite's +# completion budget (the DeviceRunner emits END_MARKER when the last test +# finishes and its screenshot has been delivered). 60s left almost no margin: +# adding a couple of GPU screenshot tests (each ~several seconds of readback on +# the emulator's software rasterizer) pushed completion just past it, so the +# marker never arrived in time and the final tests' screenshots were reported +# missing. Wait long enough that suite growth and slow runners keep margin; a +# suite that finishes sooner still breaks out immediately on marker detection, +# and a genuine hang is still caught by the missing-screenshot count guard. +TIMEOUT_SECONDS=240 START_TIME="$(date +%s)" ra_log "Waiting for DeviceRunner completion marker ($END_MARKER)" while true; do diff --git a/scripts/run-ios-native-tests.sh b/scripts/run-ios-native-tests.sh index 9a6c65f8993..a194c170dd8 100755 --- a/scripts/run-ios-native-tests.sh +++ b/scripts/run-ios-native-tests.sh @@ -82,7 +82,19 @@ if ! xcodebuild "$XCODE_CONTAINER_FLAG" "$WORKSPACE_PATH" -scheme "$TEST_SCHEME" | grep -q "platform:iOS Simulator"; then if [ "$DOWNLOAD_PLATFORMS" = "true" ]; then ri_log "No iOS simulator platform for the active Xcode; downloading via xcodebuild -downloadPlatform iOS" - xcodebuild -downloadPlatform iOS || true + # On some runner instances the CoreSimulator service is wedged and the + # download fails with "Unable to connect to simulator", leaving the job + # to fail minutes later with the misleading "Unable to find a + # destination". Detect that, restart the service and retry once. + DOWNLOAD_OUT="$(xcodebuild -downloadPlatform iOS 2>&1)" || true + printf '%s\n' "$DOWNLOAD_OUT" + if printf '%s' "$DOWNLOAD_OUT" | grep -qi "Unable to connect to simulator"; then + ri_log "CoreSimulator not responding; restarting the service and retrying the platform download" + killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true + sleep 5 + xcrun simctl list runtimes >/dev/null 2>&1 || true + xcodebuild -downloadPlatform iOS || true + fi else ri_log "No iOS simulator platform for the active Xcode. Set XCODE_DOWNLOAD_PLATFORMS=true to attempt auto-download." fi @@ -102,15 +114,23 @@ DESTINATION="$(xcodebuild "$XCODE_CONTAINER_FLAG" "$WORKSPACE_PATH" -scheme "$TE # we don't fail with "Unable to find a device" before tests can even run. if [ -z "$DESTINATION" ]; then ri_log "No concrete iOS Simulator destination from -showdestinations; querying simctl" + # Prefer a device on the NEWEST iOS runtime: a simctl-available device from + # an older Xcode's runtime is still rejected by xcodebuild destination + # matching ("Unable to find a destination matching ..."). EXISTING_ID="$(xcrun simctl list -j devices available 2>/dev/null \ - | python3 -c 'import json,sys + | python3 -c 'import json,sys,re data=json.load(sys.stdin) +best=None for runtime, devs in data.get("devices", {}).items(): if "iOS" not in runtime: continue + ver=[int(x) for x in re.findall(r"\d+", runtime)] for d in devs: if d.get("isAvailable") and "iPhone" in d.get("name",""): - print(d["udid"]); sys.exit(0)' 2>/dev/null || true)" + if best is None or ver > best[0]: + best=(ver, d["udid"]) +if best: + print(best[1])' 2>/dev/null || true)" if [ -n "$EXISTING_ID" ]; then ri_log "Reusing existing iPhone simulator $EXISTING_ID" DESTINATION="platform=iOS Simulator,id=$EXISTING_ID" diff --git a/scripts/run-javase-simulator-integration-tests.sh b/scripts/run-javase-simulator-integration-tests.sh index dc4578f6a5e..3b56e564a94 100755 --- a/scripts/run-javase-simulator-integration-tests.sh +++ b/scripts/run-javase-simulator-integration-tests.sh @@ -86,11 +86,23 @@ if [ -z "$SIM_SKIN_PATH" ] || [ ! -f "$SIM_SKIN_PATH" ]; then SKIN_ARCHIVE="$SKIN_CACHE_DIR/skins.tar.gz" SKIN_EXTRACT_DIR="$SKIN_CACHE_DIR/extracted" if [ ! -s "$SKIN_ARCHIVE" ]; then - SKIN_URL="$(python3 - <<'PY' + # Authenticate the release lookup when a token is available. The + # unauthenticated api.github.com quota is 60 requests/hour per IP, which + # a PR that fans out many jobs (plus re-runs) can exhaust on a shared + # runner IP, failing skin resolution with "HTTP Error 403: rate limit + # exceeded". A token raises the quota to 5000/hour and removes the flake. + SKIN_GH_TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}" + SKIN_URL="$(GH_API_TOKEN="$SKIN_GH_TOKEN" python3 - <<'PY' import json +import os import urllib.request -with urllib.request.urlopen('https://api.github.com/repos/codenameone/codenameone-skins/releases/latest') as response: +req = urllib.request.Request( + 'https://api.github.com/repos/codenameone/codenameone-skins/releases/latest') +token = os.environ.get('GH_API_TOKEN', '') +if token: + req.add_header('Authorization', 'Bearer ' + token) +with urllib.request.urlopen(req) as response: data = json.load(response) assets = data.get('assets') or [] @@ -158,7 +170,7 @@ MODES=(single multi) ACTUAL_ENTRIES=() for mode in "${MODES[@]}"; do if [ "$mode" = "single" ]; then - scenarios=(default landscape component-inspector network-monitor test-recorder) + scenarios=(default landscape component-inspector network-monitor test-recorder ar-demo) else scenarios=(default landscape) fi diff --git a/scripts/run-mac-native-ui-tests.sh b/scripts/run-mac-native-ui-tests.sh index c3ea7ab050b..09ef7f324a3 100755 --- a/scripts/run-mac-native-ui-tests.sh +++ b/scripts/run-mac-native-ui-tests.sh @@ -245,6 +245,31 @@ warm_catalyst_destination() { return 1 } +# The Catalyst destination of an SDKROOT=iphoneos target also needs the iOS +# platform content installed in the active Xcode. Some runner instances ship +# Xcode without it (the same provisioning gap run-ios-native-tests.sh handles), +# and no amount of destination warming helps -- the scheme lists only +# watchOS/tvOS destinations. Detect the missing iphoneos SDK and download the +# platform, restarting a wedged CoreSimulator service when the download fails +# with "Unable to connect to simulator". +ensure_ios_platform() { + if xcodebuild -showsdks 2>/dev/null | grep -q "iphoneos"; then + return 0 + fi + rm_log "The active Xcode has no iOS platform (required for the Mac Catalyst destination); downloading via xcodebuild -downloadPlatform iOS" + local out + out="$(xcodebuild -downloadPlatform iOS 2>&1)" || true + printf '%s\n' "$out" + if printf '%s' "$out" | grep -qi "Unable to connect to simulator"; then + rm_log "CoreSimulator not responding; restarting the service and retrying the platform download" + killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true + sleep 5 + xcrun simctl list runtimes >/dev/null 2>&1 || true + xcodebuild -downloadPlatform iOS || true + fi +} +ensure_ios_platform + BUILD_OK=0 for attempt in 1 2 3; do if [ "$attempt" -gt 1 ]; then diff --git a/scripts/windows/screenshots/Media360Panorama.png b/scripts/windows/screenshots/Media360Panorama.png new file mode 100644 index 00000000000..89057b64fd3 Binary files /dev/null and b/scripts/windows/screenshots/Media360Panorama.png differ diff --git a/scripts/windows/screenshots/Media360Panorama.tolerance b/scripts/windows/screenshots/Media360Panorama.tolerance new file mode 100644 index 00000000000..0c20268262d --- /dev/null +++ b/scripts/windows/screenshots/Media360Panorama.tolerance @@ -0,0 +1,5 @@ +# GPU 3D render: rasterizers vary slightly between GPUs / CI runners (the +# x64 and arm64 D3D backends differ by well under this); widen tolerance so +# noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0 diff --git a/scripts/windows/screenshots/VRStereoScene.png b/scripts/windows/screenshots/VRStereoScene.png new file mode 100644 index 00000000000..d6dee09c038 Binary files /dev/null and b/scripts/windows/screenshots/VRStereoScene.png differ diff --git a/scripts/windows/screenshots/VRStereoScene.tolerance b/scripts/windows/screenshots/VRStereoScene.tolerance new file mode 100644 index 00000000000..0c20268262d --- /dev/null +++ b/scripts/windows/screenshots/VRStereoScene.tolerance @@ -0,0 +1,5 @@ +# GPU 3D render: rasterizers vary slightly between GPUs / CI runners (the +# x64 and arm64 D3D backends differ by well under this); widen tolerance so +# noise does not fail the build. +maxChannelDelta=8 +maxMismatchPercent=3.0