Add pose detection

app/build.gradle.kts

@@ -54,7 +54,7 @@ dependencies {
     // CameraX View class
     implementation("androidx.camera:camera-view:1.0.0-alpha20")
 
-
+    implementation("com.google.mlkit:pose-detection:17.0.1-beta3")
     implementation("com.google.mlkit:face-detection:16.0.6")
     implementation("com.google.android.gms:play-services-mlkit-face-detection:16.1.5")
 

app/src/main/java/com/nekdenis/camera/MainActivity.kt

@@ -28,7 +28,9 @@ import androidx.compose.ui.Modifier
 import androidx.compose.ui.draw.scale
 import androidx.compose.ui.geometry.Offset
 import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.graphics.Brush
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.SolidColor
 import androidx.compose.ui.graphics.drawscope.Stroke
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.platform.LocalDensity
@@ -43,7 +45,10 @@ import com.google.android.gms.tasks.TaskExecutors
 import com.google.common.util.concurrent.ListenableFuture
 import com.google.mlkit.common.MlKitException
 import com.google.mlkit.vision.face.Face
+import com.google.mlkit.vision.pose.Pose
+import com.google.mlkit.vision.pose.PoseLandmark
 import com.nekdenis.camera.detection.FaceDetectorProcessor
+import com.nekdenis.camera.detection.PoseDetectorProcessor
 import com.nekdenis.camera.ui.theme.CameraComposeWorkshopTheme
 
 class MainActivity : ComponentActivity() {
@@ -110,13 +115,15 @@ class MainActivity : ComponentActivity() {
         val context = LocalContext.current
         var sourceInfo by remember { mutableStateOf(SourceInfo(10, 10, false)) }
         var detectedFaces by remember { mutableStateOf<List<Face>>(emptyList()) }
+        var detectedPose by remember { mutableStateOf<Pose?>(null) }
         val previewView = remember { PreviewView(context) }
         val cameraProvider = remember(sourceInfo, cameraLens) {
             ProcessCameraProvider.getInstance(context)
                 .configureCamera(
                     previewView, lifecycleOwner, cameraLens, context,
                     setSourceInfo = { sourceInfo = it },
                     onFacesDetected = { detectedFaces = it },
+                    onPoseDetected = { detectedPose = it }
                 )
         }
 
@@ -142,6 +149,7 @@ class MainActivity : ComponentActivity() {
                 {
                     CameraPreview(previewView)
                     DetectedFaces(faces = detectedFaces, sourceInfo = sourceInfo)
+                    DetectedPose(pose = detectedPose, sourceInfo = sourceInfo)
                 }
             }
         }
@@ -186,6 +194,94 @@ fun DetectedFaces(
     }
 }
 
+@Composable
+fun DetectedPose(
+    pose: Pose?,
+    sourceInfo: SourceInfo
+) {
+    if (pose != null) {
+        Canvas(modifier = Modifier.fillMaxSize()) {
+            val strokeWidth = 1.dp.toPx()
+            val whitePaint = SolidColor(Color.White)
+            val leftPaint = SolidColor(Color.Green)
+            val rightPaint = SolidColor(Color.Yellow)
+
+            val needToMirror = sourceInfo.isImageFlipped
+            val leftShoulder = pose.getPoseLandmark(PoseLandmark.LEFT_SHOULDER)
+            val rightShoulder = pose.getPoseLandmark(PoseLandmark.RIGHT_SHOULDER)
+            val leftElbow = pose.getPoseLandmark(PoseLandmark.LEFT_ELBOW)
+            val rightElbow = pose.getPoseLandmark(PoseLandmark.RIGHT_ELBOW)
+            val leftWrist = pose.getPoseLandmark(PoseLandmark.LEFT_WRIST)
+            val rightWrist = pose.getPoseLandmark(PoseLandmark.RIGHT_WRIST)
+            val leftHip = pose.getPoseLandmark(PoseLandmark.LEFT_HIP)
+            val rightHip = pose.getPoseLandmark(PoseLandmark.RIGHT_HIP)
+            val leftKnee = pose.getPoseLandmark(PoseLandmark.LEFT_KNEE)
+            val rightKnee = pose.getPoseLandmark(PoseLandmark.RIGHT_KNEE)
+            val leftAnkle = pose.getPoseLandmark(PoseLandmark.LEFT_ANKLE)
+            val rightAnkle = pose.getPoseLandmark(PoseLandmark.RIGHT_ANKLE)
+
+            val leftPinky = pose.getPoseLandmark(PoseLandmark.LEFT_PINKY)
+            val rightPinky = pose.getPoseLandmark(PoseLandmark.RIGHT_PINKY)
+            val leftIndex = pose.getPoseLandmark(PoseLandmark.LEFT_INDEX)
+            val rightIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_INDEX)
+            val leftThumb = pose.getPoseLandmark(PoseLandmark.LEFT_THUMB)
+            val rightThumb = pose.getPoseLandmark(PoseLandmark.RIGHT_THUMB)
+            val leftHeel = pose.getPoseLandmark(PoseLandmark.LEFT_HEEL)
+            val rightHeel = pose.getPoseLandmark(PoseLandmark.RIGHT_HEEL)
+            val leftFootIndex = pose.getPoseLandmark(PoseLandmark.LEFT_FOOT_INDEX)
+            val rightFootIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_FOOT_INDEX)
+
+            fun drawLine(
+                startLandmark: PoseLandmark?,
+                endLandmark: PoseLandmark?,
+                paint: Brush
+            ) {
+                if (startLandmark != null && endLandmark != null) {
+                    val startX =
+                        if (needToMirror) size.width - startLandmark.position.x else startLandmark.position.x
+                    val startY = startLandmark.position.y
+                    val endX =
+                        if (needToMirror) size.width - endLandmark.position.x else endLandmark.position.x
+                    val endY = endLandmark.position.y
+                    drawLine(
+                        brush = paint,
+                        start = Offset(startX, startY),
+                        end = Offset(endX, endY),
+                        strokeWidth = strokeWidth,
+                    )
+                }
+            }
+
+            drawLine(leftShoulder, rightShoulder, whitePaint)
+            drawLine(leftHip, rightHip, whitePaint)
+            // Left body
+            drawLine(leftShoulder, leftElbow, leftPaint)
+            drawLine(leftElbow, leftWrist, leftPaint)
+            drawLine(leftShoulder, leftHip, leftPaint)
+            drawLine(leftHip, leftKnee, leftPaint)
+            drawLine(leftKnee, leftAnkle, leftPaint)
+            drawLine(leftWrist, leftThumb, leftPaint)
+            drawLine(leftWrist, leftPinky, leftPaint)
+            drawLine(leftWrist, leftIndex, leftPaint)
+            drawLine(leftIndex, leftPinky, leftPaint)
+            drawLine(leftAnkle, leftHeel, leftPaint)
+            drawLine(leftHeel, leftFootIndex, leftPaint)
+            // Right body
+            drawLine(rightShoulder, rightElbow, rightPaint)
+            drawLine(rightElbow, rightWrist, rightPaint)
+            drawLine(rightShoulder, rightHip, rightPaint)
+            drawLine(rightHip, rightKnee, rightPaint)
+            drawLine(rightKnee, rightAnkle, rightPaint)
+            drawLine(rightWrist, rightThumb, rightPaint)
+            drawLine(rightWrist, rightPinky, rightPaint)
+            drawLine(rightWrist, rightIndex, rightPaint)
+            drawLine(rightIndex, rightPinky, rightPaint)
+            drawLine(rightAnkle, rightHeel, rightPaint)
+            drawLine(rightHeel, rightFootIndex, rightPaint)
+        }
+    }
+}
+
 @Composable
 fun Controls(
     onLensChange: () -> Unit
@@ -209,7 +305,8 @@ private fun ListenableFuture<ProcessCameraProvider>.configureCamera(
     cameraLens: Int,
     context: Context,
     setSourceInfo: (SourceInfo) -> Unit,
-    onFacesDetected: (List<Face>) -> Unit
+    onFacesDetected: (List<Face>) -> Unit,
+    onPoseDetected: (Pose) -> Unit
 ): ListenableFuture<ProcessCameraProvider> {
     addListener({
         val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraLens).build()
@@ -220,7 +317,8 @@ private fun ListenableFuture<ProcessCameraProvider>.configureCamera(
                 setSurfaceProvider(previewView.surfaceProvider)
             }
 
-        val analysis = bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected)
+        val analysis =
+            bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected, onPoseDetected)
         try {
             get().apply {
                 unbindAll()
@@ -244,13 +342,20 @@ private fun switchLens(lens: Int) = if (CameraSelector.LENS_FACING_FRONT == lens
 private fun bindAnalysisUseCase(
     lens: Int,
     setSourceInfo: (SourceInfo) -> Unit,
-    onFacesDetected: (List<Face>) -> Unit
+    onFacesDetected: (List<Face>) -> Unit,
+    onPoseDetected: (Pose) -> Unit
 ): ImageAnalysis? {
 
-    val imageProcessor = try {
+    val faceProcessor = try {
         FaceDetectorProcessor()
     } catch (e: Exception) {
-        Log.e("CAMERA", "Can not create image processor", e)
+        Log.e("CAMERA", "Can not create face processor", e)
+        return null
+    }
+    val poseProcessor = try {
+        PoseDetectorProcessor()
+    } catch (e: Exception) {
+        Log.e("CAMERA", "Can not create pose processor", e)
         return null
     }
     val builder = ImageAnalysis.Builder()
@@ -266,7 +371,8 @@ private fun bindAnalysisUseCase(
                 sourceInfoUpdated = true
             }
             try {
-                imageProcessor.processImageProxy(imageProxy, onFacesDetected)
+//                faceProcessor.processImageProxy(imageProxy, onFacesDetected)
+                poseProcessor.processImageProxy(imageProxy, onPoseDetected)
             } catch (e: MlKitException) {
                 Log.e(
                     "CAMERA", "Failed to process image. Error: " + e.localizedMessage

app/src/main/java/com/nekdenis/camera/detection/PoseDetectorProcessor.kt

@@ -0,0 +1,40 @@
+package com.nekdenis.camera.detection
+
+import android.annotation.SuppressLint
+import android.util.Log
+import androidx.camera.core.ImageProxy
+import com.google.android.gms.tasks.TaskExecutors
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.pose.Pose
+import com.google.mlkit.vision.pose.PoseDetection
+import com.google.mlkit.vision.pose.PoseDetector
+import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
+
+/** A processor to run pose detector.  */
+class PoseDetectorProcessor {
+
+  private val detector: PoseDetector
+
+  private val executor = TaskExecutors.MAIN_THREAD
+
+  init {
+    val options = PoseDetectorOptions.Builder()
+      .setDetectorMode(PoseDetectorOptions.STREAM_MODE)
+      .build()
+    detector = PoseDetection.getClient(options)
+  }
+
+  fun stop() {
+    detector.close()
+  }
+
+  @SuppressLint("UnsafeExperimentalUsageError")
+  fun processImageProxy(image: ImageProxy, onDetectionFinished: (Pose) -> Unit) {
+    detector.process(InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees))
+      .addOnSuccessListener(executor) { results: Pose -> onDetectionFinished(results) }
+      .addOnFailureListener(executor) { e: Exception ->
+        Log.e("Camera", "Error detecting pose", e)
+      }
+      .addOnCompleteListener { image.close() }
+  }
+}