feat: add integration with vision camera v5

.cspell-wordlist.txt

@@ -118,3 +118,7 @@ basemodule
 IMAGENET
 lraspp
 LRASPP
+worklet
+worklets
+BGRA
+RGBA

apps/computer-vision/app.json

@@ -25,11 +25,23 @@
         "foregroundImage": "./assets/icons/adaptive-icon.png",
         "backgroundColor": "#ffffff"
       },
-      "package": "com.anonymous.computervision"
+      "package": "com.anonymous.computervision",
+      "permissions": ["android.permission.CAMERA"]
     },
     "web": {
       "favicon": "./assets/icons/favicon.png"
     },
-    "plugins": ["expo-font", "expo-router"]
+    "plugins": [
+      "expo-font",
+      "expo-router",
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          }
+        }
+      ]
+    ]
   }
 }

apps/computer-vision/app/_layout.tsx

@@ -83,6 +83,14 @@ export default function _layout() {
             headerTitleStyle: { color: ColorPalette.primary },
           }}
         />
+        <Drawer.Screen
+          name="object_detection_live/index"
+          options={{
+            drawerLabel: 'Object Detection (Live)',
+            title: 'Object Detection (Live)',
+            headerTitleStyle: { color: ColorPalette.primary },
+          }}
+        />
         <Drawer.Screen
           name="ocr/index"
           options={{

apps/computer-vision/app/index.tsx

@@ -29,6 +29,12 @@ export default function Home() {
         >
           <Text style={styles.buttonText}>Object Detection</Text>
         </TouchableOpacity>
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => router.navigate('object_detection_live/')}
+        >
+          <Text style={styles.buttonText}>Object Detection Live</Text>
+        </TouchableOpacity>
         <TouchableOpacity
           style={styles.button}
           onPress={() => router.navigate('ocr/')}

apps/computer-vision/app/object_detection_live/index.tsx

@@ -0,0 +1,222 @@
+import React, {
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+import {
+  StatusBar,
+  StyleSheet,
+  Text,
+  TouchableOpacity,
+  View,
+} from 'react-native';
+import { useSafeAreaInsets } from 'react-native-safe-area-context';
+
+import {
+  Camera,
+  getCameraFormat,
+  Templates,
+  useCameraDevices,
+  useCameraPermission,
+  useFrameOutput,
+} from 'react-native-vision-camera';
+import { scheduleOnRN } from 'react-native-worklets';
+import {
+  Detection,
+  SSDLITE_320_MOBILENET_V3_LARGE,
+  useObjectDetection,
+} from 'react-native-executorch';
+import { GeneratingContext } from '../../context';
+import Spinner from '../../components/Spinner';
+import ColorPalette from '../../colors';
+
+export default function ObjectDetectionLiveScreen() {
+  const insets = useSafeAreaInsets();
+
+  const model = useObjectDetection({ model: SSDLITE_320_MOBILENET_V3_LARGE });
+  const { setGlobalGenerating } = useContext(GeneratingContext);
+
+  useEffect(() => {
+    setGlobalGenerating(model.isGenerating);
+  }, [model.isGenerating, setGlobalGenerating]);
+  const [detectionCount, setDetectionCount] = useState(0);
+  const [fps, setFps] = useState(0);
+  const lastFrameTimeRef = useRef(Date.now());
+
+  const cameraPermission = useCameraPermission();
+  const devices = useCameraDevices();
+  const device = devices.find((d) => d.position === 'back') ?? devices[0];
+
+  const format = useMemo(() => {
+    if (device == null) return undefined;
+    try {
+      return getCameraFormat(device, Templates.FrameProcessing);
+    } catch {
+      return undefined;
+    }
+  }, [device]);
+
+  const updateStats = useCallback((results: Detection[]) => {
+    setDetectionCount(results.length);
+    const now = Date.now();
+    const timeDiff = now - lastFrameTimeRef.current;
+    if (timeDiff > 0) {
+      setFps(Math.round(1000 / timeDiff));
+    }
+    lastFrameTimeRef.current = now;
+  }, []);
+
+  const frameOutput = useFrameOutput({
+    pixelFormat: 'rgb',
+    dropFramesWhileBusy: true,
+    onFrame(frame) {
+      'worklet';
+      if (!model.runOnFrame) {
+        frame.dispose();
+        return;
+      }
+      try {
+        const result = model.runOnFrame(frame, 0.5);
+        if (result) {
+          scheduleOnRN(updateStats, result);
+        }
+      } catch {
+        // ignore frame errors
+      } finally {
+        frame.dispose();
+      }
+    },
+  });
+
+  if (!model.isReady) {
+    return (
+      <Spinner
+        visible={!model.isReady}
+        textContent={`Loading the model ${(model.downloadProgress * 100).toFixed(0)} %`}
+      />
+    );
+  }
+
+  if (!cameraPermission.hasPermission) {
+    return (
+      <View style={styles.centered}>
+        <Text style={styles.message}>Camera access needed</Text>
+        <TouchableOpacity
+          onPress={() => cameraPermission.requestPermission()}
+          style={styles.button}
+        >
+          <Text style={styles.buttonText}>Grant Permission</Text>
+        </TouchableOpacity>
+      </View>
+    );
+  }
+
+  if (device == null) {
+    return (
+      <View style={styles.centered}>
+        <Text style={styles.message}>No camera device found</Text>
+      </View>
+    );
+  }
+
+  return (
+    <View style={styles.container}>
+      <StatusBar barStyle="light-content" translucent />
+
+      <Camera
+        style={StyleSheet.absoluteFill}
+        device={device}
+        outputs={[frameOutput]}
+        isActive={true}
+        format={format}
+      />
+
+      <View
+        style={[styles.bottomBarWrapper, { paddingBottom: insets.bottom + 12 }]}
+        pointerEvents="none"
+      >
+        <View style={styles.bottomBar}>
+          <View style={styles.statItem}>
+            <Text style={styles.statValue}>{detectionCount}</Text>
+            <Text style={styles.statLabel}>objects</Text>
+          </View>
+          <View style={styles.statDivider} />
+          <View style={styles.statItem}>
+            <Text style={styles.statValue}>{fps}</Text>
+            <Text style={styles.statLabel}>fps</Text>
+          </View>
+        </View>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    backgroundColor: 'black',
+  },
+  centered: {
+    flex: 1,
+    backgroundColor: 'black',
+    justifyContent: 'center',
+    alignItems: 'center',
+    gap: 16,
+  },
+  message: {
+    color: 'white',
+    fontSize: 18,
+  },
+  button: {
+    paddingHorizontal: 24,
+    paddingVertical: 12,
+    backgroundColor: ColorPalette.primary,
+    borderRadius: 24,
+  },
+  buttonText: {
+    color: 'white',
+    fontSize: 15,
+    fontWeight: '600',
+    letterSpacing: 0.3,
+  },
+  bottomBarWrapper: {
+    position: 'absolute',
+    bottom: 0,
+    left: 0,
+    right: 0,
+    alignItems: 'center',
+  },
+  bottomBar: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    backgroundColor: 'rgba(0, 0, 0, 0.55)',
+    borderRadius: 24,
+    paddingHorizontal: 28,
+    paddingVertical: 10,
+    gap: 24,
+  },
+  statItem: {
+    alignItems: 'center',
+  },
+  statValue: {
+    color: 'white',
+    fontSize: 22,
+    fontWeight: '700',
+    letterSpacing: -0.5,
+  },
+  statLabel: {
+    color: 'rgba(255,255,255,0.55)',
+    fontSize: 11,
+    fontWeight: '500',
+    textTransform: 'uppercase',
+    letterSpacing: 0.8,
+  },
+  statDivider: {
+    width: 1,
+    height: 32,
+    backgroundColor: 'rgba(255,255,255,0.2)',
+  },
+});

apps/computer-vision/metro.config.js

@@ -1,16 +1,11 @@
 // Learn more https://docs.expo.io/guides/customizing-metro
 const { getDefaultConfig } = require('expo/metro-config');
-const path = require('path');
-
-const monorepoRoot = path.resolve(__dirname, '../..');
 
 /** @type {import('expo/metro-config').MetroConfig} */
 const config = getDefaultConfig(__dirname);
 
 const { transformer, resolver } = config;
 
-config.watchFolders = [monorepoRoot];
-
 config.transformer = {
   ...transformer,
   babelTransformerPath: require.resolve('react-native-svg-transformer/expo'),
@@ -19,23 +14,6 @@ config.resolver = {
   ...resolver,
   assetExts: resolver.assetExts.filter((ext) => ext !== 'svg'),
   sourceExts: [...resolver.sourceExts, 'svg'],
-  nodeModulesPaths: [
-    path.resolve(__dirname, 'node_modules'),
-    path.resolve(monorepoRoot, 'node_modules'),
-  ],
-  // Always resolve react and react-native from the monorepo root so that
-  // workspace packages with their own nested node_modules (e.g.
-  // packages/react-native-executorch/node_modules/react) don't create a
-  // second React instance and trigger "Invalid hook call".
-  resolveRequest: (context, moduleName, platform) => {
-    if (moduleName === 'react' || moduleName === 'react-native') {
-      return {
-        filePath: require.resolve(moduleName, { paths: [monorepoRoot] }),
-        type: 'sourceFile',
-      };
-    }
-    return context.resolveRequest(context, moduleName, platform);
-  },
 };
 
 config.resolver.assetExts.push('pte');

apps/computer-vision/package.json

@@ -17,6 +17,7 @@
     "@react-navigation/native": "^7.1.28",
     "@shopify/react-native-skia": "2.4.21",
     "expo": "^54.0.27",
+    "expo-build-properties": "~1.0.10",
     "expo-constants": "~18.0.11",
     "expo-font": "~14.0.10",
     "expo-linking": "~8.0.10",
@@ -30,17 +31,20 @@
     "react-native-gesture-handler": "~2.28.0",
     "react-native-image-picker": "^7.2.2",
     "react-native-loading-spinner-overlay": "^3.0.1",
+    "react-native-nitro-image": "^0.12.0",
+    "react-native-nitro-modules": "^0.33.9",
     "react-native-reanimated": "~4.2.2",
     "react-native-safe-area-context": "~5.6.0",
     "react-native-screens": "~4.16.0",
     "react-native-svg": "15.15.3",
     "react-native-svg-transformer": "^1.5.3",
+    "react-native-vision-camera": "5.0.0-beta.2",
     "react-native-worklets": "0.7.4"
   },
   "devDependencies": {
     "@babel/core": "^7.29.0",
     "@types/pngjs": "^6.0.5",
-    "@types/react": "~19.1.10"
+    "@types/react": "~19.2.0"
   },
   "private": true
 }

apps/llm/metro.config.js

@@ -2,15 +2,11 @@ const { getDefaultConfig } = require('expo/metro-config');
 const {
   wrapWithAudioAPIMetroConfig,
 } = require('react-native-audio-api/metro-config');
-const path = require('path');
 
-const monorepoRoot = path.resolve(__dirname, '../..');
 const config = getDefaultConfig(__dirname);
 
 const { transformer, resolver } = config;
 
-config.watchFolders = [monorepoRoot];
-
 config.transformer = {
   ...transformer,
   babelTransformerPath: require.resolve('react-native-svg-transformer/expo'),
@@ -19,23 +15,6 @@ config.resolver = {
   ...resolver,
   assetExts: resolver.assetExts.filter((ext) => ext !== 'svg'),
   sourceExts: [...resolver.sourceExts, 'svg'],
-  nodeModulesPaths: [
-    path.resolve(__dirname, 'node_modules'),
-    path.resolve(monorepoRoot, 'node_modules'),
-  ],
-  // Always resolve react and react-native from the monorepo root so that
-  // workspace packages with their own nested node_modules (e.g.
-  // packages/react-native-executorch/node_modules/react) don't create a
-  // second React instance and trigger "Invalid hook call".
-  resolveRequest: (context, moduleName, platform) => {
-    if (moduleName === 'react' || moduleName === 'react-native') {
-      return {
-        filePath: require.resolve(moduleName, { paths: [monorepoRoot] }),
-        type: 'sourceFile',
-      };
-    }
-    return context.resolveRequest(context, moduleName, platform);
-  },
 };
 
 config.resolver.assetExts.push('pte');