diff --git a/app/build.gradle.kts b/app/build.gradle.kts index e8e37c3..105437c 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -54,7 +54,7 @@ dependencies { // CameraX View class implementation("androidx.camera:camera-view:1.0.0-alpha20") - + implementation("com.google.mlkit:pose-detection:17.0.1-beta3") implementation("com.google.mlkit:face-detection:16.0.6") implementation("com.google.android.gms:play-services-mlkit-face-detection:16.1.5") diff --git a/app/src/main/java/com/nekdenis/camera/MainActivity.kt b/app/src/main/java/com/nekdenis/camera/MainActivity.kt index 9b67879..818e3af 100644 --- a/app/src/main/java/com/nekdenis/camera/MainActivity.kt +++ b/app/src/main/java/com/nekdenis/camera/MainActivity.kt @@ -28,7 +28,9 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.draw.scale import androidx.compose.ui.geometry.Offset import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor import androidx.compose.ui.graphics.drawscope.Stroke import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalDensity @@ -43,7 +45,10 @@ import com.google.android.gms.tasks.TaskExecutors import com.google.common.util.concurrent.ListenableFuture import com.google.mlkit.common.MlKitException import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseLandmark import com.nekdenis.camera.detection.FaceDetectorProcessor +import com.nekdenis.camera.detection.PoseDetectorProcessor import com.nekdenis.camera.ui.theme.CameraComposeWorkshopTheme class MainActivity : ComponentActivity() { @@ -110,6 +115,7 @@ class MainActivity : ComponentActivity() { val context = LocalContext.current var sourceInfo by remember { mutableStateOf(SourceInfo(10, 10, false)) } var detectedFaces by remember { mutableStateOf>(emptyList()) } + var detectedPose by remember { mutableStateOf(null) } val previewView = remember { PreviewView(context) } val cameraProvider = remember(sourceInfo, cameraLens) { ProcessCameraProvider.getInstance(context) @@ -117,6 +123,7 @@ class MainActivity : ComponentActivity() { previewView, lifecycleOwner, cameraLens, context, setSourceInfo = { sourceInfo = it }, onFacesDetected = { detectedFaces = it }, + onPoseDetected = { detectedPose = it } ) } @@ -142,6 +149,7 @@ class MainActivity : ComponentActivity() { { CameraPreview(previewView) DetectedFaces(faces = detectedFaces, sourceInfo = sourceInfo) + DetectedPose(pose = detectedPose, sourceInfo = sourceInfo) } } } @@ -186,6 +194,94 @@ fun DetectedFaces( } } +@Composable +fun DetectedPose( + pose: Pose?, + sourceInfo: SourceInfo +) { + if (pose != null) { + Canvas(modifier = Modifier.fillMaxSize()) { + val strokeWidth = 1.dp.toPx() + val whitePaint = SolidColor(Color.White) + val leftPaint = SolidColor(Color.Green) + val rightPaint = SolidColor(Color.Yellow) + + val needToMirror = sourceInfo.isImageFlipped + val leftShoulder = pose.getPoseLandmark(PoseLandmark.LEFT_SHOULDER) + val rightShoulder = pose.getPoseLandmark(PoseLandmark.RIGHT_SHOULDER) + val leftElbow = pose.getPoseLandmark(PoseLandmark.LEFT_ELBOW) + val rightElbow = pose.getPoseLandmark(PoseLandmark.RIGHT_ELBOW) + val leftWrist = pose.getPoseLandmark(PoseLandmark.LEFT_WRIST) + val rightWrist = pose.getPoseLandmark(PoseLandmark.RIGHT_WRIST) + val leftHip = pose.getPoseLandmark(PoseLandmark.LEFT_HIP) + val rightHip = pose.getPoseLandmark(PoseLandmark.RIGHT_HIP) + val leftKnee = pose.getPoseLandmark(PoseLandmark.LEFT_KNEE) + val rightKnee = pose.getPoseLandmark(PoseLandmark.RIGHT_KNEE) + val leftAnkle = pose.getPoseLandmark(PoseLandmark.LEFT_ANKLE) + val rightAnkle = pose.getPoseLandmark(PoseLandmark.RIGHT_ANKLE) + + val leftPinky = pose.getPoseLandmark(PoseLandmark.LEFT_PINKY) + val rightPinky = pose.getPoseLandmark(PoseLandmark.RIGHT_PINKY) + val leftIndex = pose.getPoseLandmark(PoseLandmark.LEFT_INDEX) + val rightIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_INDEX) + val leftThumb = pose.getPoseLandmark(PoseLandmark.LEFT_THUMB) + val rightThumb = pose.getPoseLandmark(PoseLandmark.RIGHT_THUMB) + val leftHeel = pose.getPoseLandmark(PoseLandmark.LEFT_HEEL) + val rightHeel = pose.getPoseLandmark(PoseLandmark.RIGHT_HEEL) + val leftFootIndex = pose.getPoseLandmark(PoseLandmark.LEFT_FOOT_INDEX) + val rightFootIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_FOOT_INDEX) + + fun drawLine( + startLandmark: PoseLandmark?, + endLandmark: PoseLandmark?, + paint: Brush + ) { + if (startLandmark != null && endLandmark != null) { + val startX = + if (needToMirror) size.width - startLandmark.position.x else startLandmark.position.x + val startY = startLandmark.position.y + val endX = + if (needToMirror) size.width - endLandmark.position.x else endLandmark.position.x + val endY = endLandmark.position.y + drawLine( + brush = paint, + start = Offset(startX, startY), + end = Offset(endX, endY), + strokeWidth = strokeWidth, + ) + } + } + + drawLine(leftShoulder, rightShoulder, whitePaint) + drawLine(leftHip, rightHip, whitePaint) + // Left body + drawLine(leftShoulder, leftElbow, leftPaint) + drawLine(leftElbow, leftWrist, leftPaint) + drawLine(leftShoulder, leftHip, leftPaint) + drawLine(leftHip, leftKnee, leftPaint) + drawLine(leftKnee, leftAnkle, leftPaint) + drawLine(leftWrist, leftThumb, leftPaint) + drawLine(leftWrist, leftPinky, leftPaint) + drawLine(leftWrist, leftIndex, leftPaint) + drawLine(leftIndex, leftPinky, leftPaint) + drawLine(leftAnkle, leftHeel, leftPaint) + drawLine(leftHeel, leftFootIndex, leftPaint) + // Right body + drawLine(rightShoulder, rightElbow, rightPaint) + drawLine(rightElbow, rightWrist, rightPaint) + drawLine(rightShoulder, rightHip, rightPaint) + drawLine(rightHip, rightKnee, rightPaint) + drawLine(rightKnee, rightAnkle, rightPaint) + drawLine(rightWrist, rightThumb, rightPaint) + drawLine(rightWrist, rightPinky, rightPaint) + drawLine(rightWrist, rightIndex, rightPaint) + drawLine(rightIndex, rightPinky, rightPaint) + drawLine(rightAnkle, rightHeel, rightPaint) + drawLine(rightHeel, rightFootIndex, rightPaint) + } + } +} + @Composable fun Controls( onLensChange: () -> Unit @@ -209,7 +305,8 @@ private fun ListenableFuture.configureCamera( cameraLens: Int, context: Context, setSourceInfo: (SourceInfo) -> Unit, - onFacesDetected: (List) -> Unit + onFacesDetected: (List) -> Unit, + onPoseDetected: (Pose) -> Unit ): ListenableFuture { addListener({ val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraLens).build() @@ -220,7 +317,8 @@ private fun ListenableFuture.configureCamera( setSurfaceProvider(previewView.surfaceProvider) } - val analysis = bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected) + val analysis = + bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected, onPoseDetected) try { get().apply { unbindAll() @@ -244,13 +342,20 @@ private fun switchLens(lens: Int) = if (CameraSelector.LENS_FACING_FRONT == lens private fun bindAnalysisUseCase( lens: Int, setSourceInfo: (SourceInfo) -> Unit, - onFacesDetected: (List) -> Unit + onFacesDetected: (List) -> Unit, + onPoseDetected: (Pose) -> Unit ): ImageAnalysis? { - val imageProcessor = try { + val faceProcessor = try { FaceDetectorProcessor() } catch (e: Exception) { - Log.e("CAMERA", "Can not create image processor", e) + Log.e("CAMERA", "Can not create face processor", e) + return null + } + val poseProcessor = try { + PoseDetectorProcessor() + } catch (e: Exception) { + Log.e("CAMERA", "Can not create pose processor", e) return null } val builder = ImageAnalysis.Builder() @@ -266,7 +371,8 @@ private fun bindAnalysisUseCase( sourceInfoUpdated = true } try { - imageProcessor.processImageProxy(imageProxy, onFacesDetected) +// faceProcessor.processImageProxy(imageProxy, onFacesDetected) + poseProcessor.processImageProxy(imageProxy, onPoseDetected) } catch (e: MlKitException) { Log.e( "CAMERA", "Failed to process image. Error: " + e.localizedMessage diff --git a/app/src/main/java/com/nekdenis/camera/detection/PoseDetectorProcessor.kt b/app/src/main/java/com/nekdenis/camera/detection/PoseDetectorProcessor.kt new file mode 100644 index 0000000..ce9f7ee --- /dev/null +++ b/app/src/main/java/com/nekdenis/camera/detection/PoseDetectorProcessor.kt @@ -0,0 +1,40 @@ +package com.nekdenis.camera.detection + +import android.annotation.SuppressLint +import android.util.Log +import androidx.camera.core.ImageProxy +import com.google.android.gms.tasks.TaskExecutors +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.PoseDetector +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions + +/** A processor to run pose detector. */ +class PoseDetectorProcessor { + + private val detector: PoseDetector + + private val executor = TaskExecutors.MAIN_THREAD + + init { + val options = PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) + .build() + detector = PoseDetection.getClient(options) + } + + fun stop() { + detector.close() + } + + @SuppressLint("UnsafeExperimentalUsageError") + fun processImageProxy(image: ImageProxy, onDetectionFinished: (Pose) -> Unit) { + detector.process(InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees)) + .addOnSuccessListener(executor) { results: Pose -> onDetectionFinished(results) } + .addOnFailureListener(executor) { e: Exception -> + Log.e("Camera", "Error detecting pose", e) + } + .addOnCompleteListener { image.close() } + } +} \ No newline at end of file