Text Recognition

.changeset/nine-ads-drum.md

@@ -0,0 +1,6 @@
+---
+"@infinitered/react-native-mlkit-text-recognition": major
+"example-app": major
+---
+
+Added first version of Text Recognition module

.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "WebFetch(domain:developers.google.com)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

apps/ExampleApp/app.json

@@ -18,9 +18,7 @@
       "fallbackToCacheTimeout": 0
     },
     "jsEngine": "hermes",
-    "assetBundlePatterns": [
-      "**/*"
-    ],
+    "assetBundlePatterns": ["**/*"],
     "android": {
       "icon": "./assets/images/app-icon-android-legacy.png",
       "package": "red.infinite.reactnativemlkit.example",
@@ -64,6 +62,12 @@
           "backgroundColor": "#F4F2F1",
           "imageWidth": 250
         }
+      ],
+      [
+        "expo-image-picker",
+        {
+          "photosPermission": "This app uses the photo library to select images for Machine Learning purposes. i.e. Object and Image detection."
+        }
       ]
     ],
     "experiments": {

apps/ExampleApp/app/navigators/AppNavigator.tsx

@@ -35,6 +35,7 @@ export type AppStackParamList = {
   ImageLabeling: Record<string, never>
   ObjectDetection: Record<string, never>
   DocumentScanner: Record<string, never>
+  TextRecognition: Record<string, never>
   // IGNITE_GENERATOR_ANCHOR_APP_STACK_PARAM_LIST
 }
 
@@ -68,6 +69,7 @@ const AppStack = observer(function AppStack() {
       <Stack.Screen name="ImageLabeling" component={Screens.ImageLabelingScreen} />
       <Stack.Screen name="ObjectDetection" component={Screens.ObjectDetectionScreen} />
       <Stack.Screen name="DocumentScanner" component={Screens.DocumentScannerScreen} />
+      <Stack.Screen name="TextRecognition" component={Screens.TextRecognitionScreen} />
       {/* IGNITE_GENERATOR_ANCHOR_APP_STACK_SCREENS */}
     </Stack.Navigator>
   )

apps/ExampleApp/app/screens/HomeScreen/demoInfo.ts

@@ -11,6 +11,7 @@ export interface DemoInfo {
 const FACE_DETECTION = require("../../../assets/images/face-detection.jpg")
 const FACE_HOLDER = require("../../../assets/images/welcome-face.png")
 const DOCUMENT_SCANNER = require("../../../assets/images/doc-scanner.png")
+const TEXT_RECOGNITION = require("../../../assets/images/text-recognition.png")
 
 const ANDROID_ONLY_DEMOS: DemoInfo[] = [
   {
@@ -46,5 +47,11 @@ export const DEMO_LIST: DemoInfo[] = [
     screen: "ImageLabeling",
     image: FACE_HOLDER,
   },
+  {
+    title: "Text recognition",
+    description: "Recognize text in an image",
+    screen: "TextRecognition",
+    image: TEXT_RECOGNITION,
+  },
   ...PLATFORM_SPECIFIC_DEMOS,
 ]

apps/ExampleApp/app/screens/TextRecognitionScreen.tsx

@@ -0,0 +1,130 @@
+import React, { FC, useState, useEffect, useCallback } from "react"
+import { observer } from "mobx-react-lite"
+import { ViewStyle, View, ImageStyle, TextStyle } from "react-native"
+import { NativeStackScreenProps } from "@react-navigation/native-stack"
+import { AppStackScreenProps } from "../navigators"
+import { Text, Icon, ImageSelector, Screen } from "../components"
+import { useTypedNavigation } from "../navigators/useTypedNavigation"
+
+import { recognizeText } from "@infinitered/react-native-mlkit-text-recognition"
+import { UseExampleImageStatus, SelectedImage } from "../utils/useExampleImage"
+
+type TextRecognitionScreenProps = NativeStackScreenProps<AppStackScreenProps<"TextRecognition">>
+
+export const TextRecognitionScreen: FC<TextRecognitionScreenProps> = observer(
+  function TextRecognitionScreen() {
+    const navigation = useTypedNavigation<"TextRecognition">()
+
+    const [image, setImage] = useState<SelectedImage | null>(null)
+
+    const handleImageChange = useCallback((nextImage: SelectedImage) => {
+      setImage(nextImage)
+    }, [])
+
+    const [result, setResult] = useState<string | null>(null)
+    const [status, setStatus] = useState<
+      "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus
+    >("init")
+
+    const onStatusChange = React.useCallback(
+      (status: "init" | "noPermissions" | "done" | "error" | "loading" | UseExampleImageStatus) => {
+        setStatus(status)
+      },
+      [],
+    )
+
+    useEffect(() => {
+      const recognizeImage = async () => {
+        if (!image?.uri) return
+        setStatus("recognizing")
+        try {
+          const recognitionResult = await recognizeText(image.uri)
+          setResult(recognitionResult.text)
+          setStatus("done")
+        } catch (error) {
+          console.error("Error recognizing image:", error)
+          setStatus("error")
+        }
+      }
+
+      recognizeImage().then(() => null)
+    }, [image])
+
+    const statusMessage = React.useMemo(() => {
+      if (!image && status !== "init") {
+        setStatus("init")
+      }
+      switch (status) {
+        case "init":
+          return "Take a photo or select one from your camera roll"
+        case "noPermissions":
+          return "You need to grant camera permissions to take a photo"
+        case "takingPhoto":
+          return "Taking photo..."
+        case "selectingPhoto":
+          return "Selecting photo..."
+        case "done":
+          return "Done!"
+        case "error":
+          return "Error during recognition!"
+        case "recognizing":
+          return "Recognizing Image..."
+        case "loading":
+          return "Loading Example Images..."
+        default:
+          throw new Error("Invalid status")
+      }
+    }, [result, image, status])
+
+    const clearResults = useCallback(() => {
+      setResult(null)
+    }, [])
+
+    return (
+      <Screen style={$root} preset="scroll" safeAreaEdges={["top", "bottom"]}>
+        <View>
+          <Icon icon={"back"} onPress={() => navigation.navigate("Home")} style={$backIcon} />
+          <Text preset={"heading"} text="Text Recognition" />
+          <Text style={$description}>Take a photo, and extract text from it.</Text>
+        </View>
+        <ImageSelector
+          onImageChange={handleImageChange}
+          onImageClear={clearResults}
+          onStatusChange={onStatusChange}
+          statusMessage={statusMessage}
+          status={status}
+          isLoading={false}
+          images={{
+            filter: "all",
+            groupBy: "label",
+          }}
+        />
+
+        {result && (
+          <View style={$resultContainer}>
+            <Text>{result}</Text>
+          </View>
+        )}
+      </Screen>
+    )
+  },
+)
+
+const $root: ViewStyle = {
+  flex: 1,
+  padding: 16,
+  display: "flex",
+  flexDirection: "column",
+}
+const $backIcon: ImageStyle = { marginVertical: 8 }
+
+const $description: TextStyle = {
+  marginVertical: 8,
+  color: "rgba(0,0,0,0.6)",
+}
+
+const $resultContainer: ViewStyle = {
+  width: "100%",
+  borderWidth: 1,
+  marginVertical: 24,
+}

apps/ExampleApp/app/screens/index.ts

@@ -7,3 +7,4 @@ export * from "./ImageLabelingScreen"
 export * from "./DocumentScannerScreen"
 export { BOX_COLORS } from "./FaceDetectionScreen"
 export * from "./ObjectDetectionScreen"
+export * from "./TextRecognitionScreen"

apps/ExampleApp/app/utils/useExampleImage/useExampleImage.ts

@@ -1,9 +1,7 @@
 import {
   ImagePickerResult,
-  launchCameraAsync,
   launchImageLibraryAsync,
   ImagePickerOptions,
-  MediaTypeOptions,
   ImagePickerAsset,
   useCameraPermissions,
 } from "expo-image-picker"
@@ -17,13 +15,15 @@ import {
   imageFilters,
   imageGroupers,
 } from "./examples"
+import { launchImageLibrary } from "react-native-image-picker"
 
 export type UseExampleImageStatus =
   | "init"
   | "noPermissions"
   | "takingPhoto"
   | "selectingPhoto"
   | "classifying"
+  | "recognizing"
   | "done"
   | "error"
   | "loading"
@@ -56,7 +56,7 @@ interface ZippedImage extends RandomImage {
 }
 
 const IMAGE_PICKER_OPTIONS: ImagePickerOptions = {
-  mediaTypes: MediaTypeOptions.Images,
+  mediaTypes: "images",
   allowsEditing: false,
   quality: 0.5,
 }
@@ -127,7 +127,9 @@ export function useExampleImage(predicates?: {
       return
     }
     setStatus("takingPhoto")
-    const result = await launchCameraAsync(IMAGE_PICKER_OPTIONS)
+    const result = await launchImageLibrary({
+      mediaType: "photo",
+    })
     if (result.assets?.[0]) {
       setImage({ ...result.assets?.[0], localUri: result.assets?.[0].uri } as SelectedImage)
     } else {

apps/ExampleApp/package.json

@@ -36,34 +36,36 @@
     "@infinitered/react-native-mlkit-face-detection": "workspace:^3.1.0",
     "@infinitered/react-native-mlkit-image-labeling": "workspace:^3.1.0",
     "@infinitered/react-native-mlkit-object-detection": "workspace:^3.1.0",
-    "@react-native-async-storage/async-storage": "^2.0.0",
+    "@infinitered/react-native-mlkit-text-recognition": "workspace:^1.0.0",
+    "@react-native-async-storage/async-storage": "1.23.1",
     "@react-navigation/native": "^6.0.8",
     "@react-navigation/native-stack": "^6.0.2",
-    "@shopify/flash-list": "1.7.1",
+    "@shopify/flash-list": "1.7.3",
     "apisauce": "3.0.1",
     "date-fns": "^2.30.0",
-    "expo": "^52.0.28",
-    "expo-build-properties": "~0.13.2",
-    "expo-dev-client": "~5.0.11",
-    "expo-font": "~13.0.3",
-    "expo-image": "~2.0.4",
-    "expo-image-picker": "~16.0.5",
+    "expo": "~52.0.47",
+    "expo-build-properties": "~0.13.3",
+    "expo-dev-client": "~5.0.20",
+    "expo-font": "~13.0.4",
+    "expo-image": "~2.0.7",
+    "expo-image-picker": "~16.0.6",
     "expo-linking": "~7.0.5",
     "expo-localization": "~16.0.1",
-    "expo-splash-screen": "~0.29.21",
+    "expo-splash-screen": "~0.29.24",
     "expo-status-bar": "~2.0.1",
     "i18n-js": "3.9.2",
     "mobx": "6.10.2",
     "mobx-react-lite": "4.0.5",
     "mobx-state-tree": "5.3.0",
     "react": "18.3.1",
     "react-dom": "^18.3.1",
-    "react-native": "^0.76.0",
+    "react-native": "0.76.9",
     "react-native-device-info": "^14.0.4",
-    "react-native-gesture-handler": "^2.20.0",
+    "react-native-gesture-handler": "~2.20.2",
+    "react-native-image-picker": "^8.2.1",
     "react-native-reanimated": "^3.16.1",
-    "react-native-safe-area-context": "^4.12.0",
-    "react-native-screens": "^3.34.0",
+    "react-native-safe-area-context": "4.12.0",
+    "react-native-screens": "~4.4.0",
     "react-native-web": "~0.19.13"
   },
   "devDependencies": {
@@ -100,7 +102,7 @@
     "eslint-plugin-reactotron": "^0.1.2",
     "expo-atlas": "^0.3.0",
     "jest": "^29.2.1",
-    "jest-expo": "~52.0.3",
+    "jest-expo": "~52.0.6",
     "patch-package": "6.4.7",
     "postinstall-prepare": "1.0.1",
     "prettier": "2.8.8",
@@ -111,7 +113,7 @@
     "reactotron-react-native": "^5.0.5",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.2",
-    "typescript": "~5.1.6"
+    "typescript": "^5.3.3"
   },
   "engines": {
     "node": ">=18"

docs/text-recognition/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Text Recognition",
+  "position": 500,
+  "link": {
+    "type": "generated-index",
+    "description": "Recognize text"
+  }
+}