@nk/vision models camera integration

.cspell-wordlist.txt

@@ -116,3 +116,6 @@ antonov
 rfdetr
 basemodule
 IMAGENET
+worklet
+worklets
+BGRA

apps/computer-vision/app.json

@@ -25,11 +25,23 @@
         "foregroundImage": "./assets/icons/adaptive-icon.png",
         "backgroundColor": "#ffffff"
       },
-      "package": "com.anonymous.computervision"
+      "package": "com.anonymous.computervision",
+      "permissions": ["android.permission.CAMERA"]
     },
     "web": {
       "favicon": "./assets/icons/favicon.png"
     },
-    "plugins": ["expo-font", "expo-router"]
+    "plugins": [
+      "expo-font",
+      "expo-router",
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          }
+        }
+      ]
+    ]
   }
 }

apps/computer-vision/app/_layout.tsx

@@ -59,6 +59,15 @@ export default function _layout() {
           headerTitleStyle: { color: ColorPalette.primary },
         }}
       >
+        <Drawer.Screen
+          name="vision_camera/index"
+          options={{
+            drawerLabel: 'Vision Camera',
+            title: 'Vision Camera',
+            headerShown: false,
+            headerTitleStyle: { color: ColorPalette.primary },
+          }}
+        />
         <Drawer.Screen
           name="classification/index"
           options={{

apps/computer-vision/app/index.tsx

@@ -11,6 +11,12 @@ export default function Home() {
       <ExecutorchLogo width={64} height={64} />
       <Text style={styles.headerText}>Select a demo model</Text>
       <View style={styles.buttonContainer}>
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => router.navigate('vision_camera/')}
+        >
+          <Text style={styles.buttonText}>Vision Camera</Text>
+        </TouchableOpacity>
         <TouchableOpacity
           style={styles.button}
           onPress={() => router.navigate('classification/')}

apps/computer-vision/app/style_transfer/index.tsx

@@ -5,6 +5,14 @@ import {
   useStyleTransfer,
   STYLE_TRANSFER_CANDY,
 } from 'react-native-executorch';
+import {
+  Canvas,
+  Image as SkiaImage,
+  Skia,
+  AlphaType,
+  ColorType,
+  SkImage,
+} from '@shopify/react-native-skia';
 import { View, StyleSheet, Image } from 'react-native';
 import React, { useContext, useEffect, useState } from 'react';
 import { GeneratingContext } from '../../context';
@@ -16,20 +24,38 @@ export default function StyleTransferScreen() {
   useEffect(() => {
     setGlobalGenerating(model.isGenerating);
   }, [model.isGenerating, setGlobalGenerating]);
+
   const [imageUri, setImageUri] = useState('');
+  const [styledImage, setStyledImage] = useState<SkImage | null>(null);
+  const [canvasSize, setCanvasSize] = useState({ width: 1, height: 1 });
+
   const handleCameraPress = async (isCamera: boolean) => {
     const image = await getImage(isCamera);
     const uri = image?.uri;
     if (typeof uri === 'string') {
-      setImageUri(uri as string);
+      setImageUri(uri);
+      setStyledImage(null);
     }
   };
 
   const runForward = async () => {
     if (imageUri) {
       try {
         const output = await model.forward(imageUri);
-        setImageUri(output);
+        const height = output.sizes[0];
+        const width = output.sizes[1];
+        const skData = Skia.Data.fromBytes(output.dataPtr);
+        const img = Skia.Image.MakeImage(
+          {
+            width,
+            height,
+            alphaType: AlphaType.Opaque,
+            colorType: ColorType.RGBA_8888,
+          },
+          skData,
+          width * 4
+        );
+        setStyledImage(img);
       } catch (e) {
         console.error(e);
       }
@@ -48,15 +74,38 @@ export default function StyleTransferScreen() {
   return (
     <ScreenWrapper>
       <View style={styles.imageContainer}>
-        <Image
-          style={styles.image}
-          resizeMode="contain"
-          source={
-            imageUri
-              ? { uri: imageUri }
-              : require('../../assets/icons/executorch_logo.png')
-          }
-        />
+        {styledImage ? (
+          <View
+            style={styles.canvas}
+            onLayout={(e) =>
+              setCanvasSize({
+                width: e.nativeEvent.layout.width,
+                height: e.nativeEvent.layout.height,
+              })
+            }
+          >
+            <Canvas style={StyleSheet.absoluteFill}>
+              <SkiaImage
+                image={styledImage}
+                fit="contain"
+                x={0}
+                y={0}
+                width={canvasSize.width}
+                height={canvasSize.height}
+              />
+            </Canvas>
+          </View>
+        ) : (
+          <Image
+            style={styles.image}
+            resizeMode="contain"
+            source={
+              imageUri
+                ? { uri: imageUri }
+                : require('../../assets/icons/executorch_logo.png')
+            }
+          />
+        )}
       </View>
       <BottomBar
         handleCameraPress={handleCameraPress}
@@ -77,4 +126,8 @@ const styles = StyleSheet.create({
     borderRadius: 8,
     width: '100%',
   },
+  canvas: {
+    flex: 1,
+    width: '100%',
+  },
 });