From 313cea67e97063a78ce3962419f64a773481c4b7 Mon Sep 17 00:00:00 2001 From: Hannes Achleitner Date: Wed, 17 Mar 2021 07:34:43 +0100 Subject: [PATCH 1/4] queueEvent --- .../example/camerax/tflite/CameraActivity.kt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt index fd270a0..1759e9f 100644 --- a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt +++ b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt @@ -27,6 +27,7 @@ import android.media.Image import android.opengl.GLES20 import android.opengl.GLSurfaceView import android.os.Bundle +import android.util.Log import android.util.Size import android.view.View import android.view.ViewGroup @@ -156,8 +157,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { } /** Declare and bind preview and analysis use cases */ - @SuppressLint("UnsafeExperimentalUsageError") - private fun bindCameraUseCases() = view_finder.post { + private fun bindCameraUseCases() { val cameraProviderFuture = ProcessCameraProvider.getInstance(this) cameraProviderFuture.addListener({ @@ -174,6 +174,8 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { // Obtain the current frame from ARSession. When the configuration is set to // UpdateMode.BLOCKING (it is by default), this will throttle the rendering to the // camera framerate. + Log.w("sessionUpdate", Thread.currentThread().name) + val frame: Frame = session.update() val image: Image try { @@ -213,7 +215,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { val predictions = detector.predict(tfImage) // Report only the top prediction - reportPrediction(predictions.maxBy { it.score }) + surfaceView.queueEvent { reportPrediction(predictions.maxByOrNull { it.score }) } /*imageAnalysis.setAnalyzer(executor, ImageAnalysis.Analyzer { image -> if (!::bitmapBuffer.isInitialized) { @@ -271,13 +273,13 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { }, ContextCompat.getMainExecutor(this)) } - private fun reportPrediction(prediction: ObjectDetectionHelper.ObjectPrediction?) = view_finder.post { + private fun reportPrediction(prediction: ObjectDetectionHelper.ObjectPrediction?) { // Early exit: if prediction is not good enough, don't report it if (prediction == null || prediction.score < ACCURACY_THRESHOLD) { box_prediction.visibility = View.GONE text_prediction.visibility = View.GONE - return@post + return } // Location has to be mapped to our local coordinates @@ -360,7 +362,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { if (!hasPermissions(this)) { ActivityCompat.requestPermissions(this, permissions.toTypedArray(), permissionsRequestCode) } else { - bindCameraUseCases() + surfaceView.queueEvent { bindCameraUseCases() } } session.resume() surfaceView.onResume() @@ -382,7 +384,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { super.onRequestPermissionsResult(requestCode, permissions, grantResults) if (requestCode == permissionsRequestCode && hasPermissions(this)) { - bindCameraUseCases() + surfaceView.queueEvent { bindCameraUseCases() } } else { finish() // If we don't have the required permissions, we can't run } @@ -423,6 +425,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { } override fun onDrawFrame(p0: GL10?) { + Log.w("sessionDraw", Thread.currentThread().name) val frame = session.update() From 511875855bdc59e8952e32c42c6b38bd0c23b7fb Mon Sep 17 00:00:00 2001 From: Hannes Achleitner Date: Thu, 18 Mar 2021 10:51:12 +0100 Subject: [PATCH 2/4] Dirty way to set right thread --- .../example/camerax/tflite/CameraActivity.kt | 145 +++++++++--------- 1 file changed, 75 insertions(+), 70 deletions(-) diff --git a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt index 1759e9f..b0ccb38 100644 --- a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt +++ b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt @@ -17,13 +17,11 @@ package com.android.example.camerax.tflite import android.Manifest -import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Matrix import android.graphics.RectF -import android.media.Image import android.opengl.GLES20 import android.opengl.GLSurfaceView import android.os.Bundle @@ -32,13 +30,18 @@ import android.util.Size import android.view.View import android.view.ViewGroup import androidx.appcompat.app.AppCompatActivity -import androidx.camera.core.* +import androidx.camera.core.AspectRatio +import androidx.camera.core.CameraSelector +import androidx.camera.core.Preview import androidx.camera.lifecycle.ProcessCameraProvider import androidx.constraintlayout.widget.ConstraintLayout import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import androidx.lifecycle.LifecycleOwner -import com.google.ar.core.* +import com.google.ar.core.Config +import com.google.ar.core.Frame +import com.google.ar.core.InstantPlacementPoint +import com.google.ar.core.Session import com.google.ar.core.exceptions.NotYetAvailableException import kotlinx.android.synthetic.main.activity_camera.* import org.tensorflow.lite.DataType @@ -61,7 +64,7 @@ import kotlin.random.Random class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { private lateinit var container: ConstraintLayout - private lateinit var bitmapBuffer: Bitmap + private var bitmapBuffer: Bitmap? = null private lateinit var surfaceView: GLSurfaceView private val permissions = listOf(Manifest.permission.CAMERA) @@ -80,17 +83,19 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { private val tfImageProcessor by lazy { - val cropSize = minOf(bitmapBuffer.width, bitmapBuffer.height) - ImageProcessor.Builder() - .add(ResizeWithCropOrPadOp(cropSize, cropSize)) - .add( - ResizeOp( - tfInputSize.height, tfInputSize.width, ResizeOp.ResizeMethod.NEAREST_NEIGHBOR + bitmapBuffer?.let { + val cropSize = minOf(it.width, it.height) + ImageProcessor.Builder() + .add(ResizeWithCropOrPadOp(cropSize, cropSize)) + .add( + ResizeOp( + tfInputSize.height, tfInputSize.width, ResizeOp.ResizeMethod.NEAREST_NEIGHBOR + ) ) - ) - .add(Rot90Op(imageRotationDegrees / 90)) - .add(NormalizeOp(0f, 1f)) - .build() + .add(Rot90Op(imageRotationDegrees / 90)) + .add(NormalizeOp(0f, 1f)) + .build() + } } private val tflite by lazy { @@ -143,10 +148,10 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { postRotate(imageRotationDegrees.toFloat()) if (isFrontFacing) postScale(-1f, 1f) } - val uprightImage = Bitmap.createBitmap( - bitmapBuffer, 0, 0, bitmapBuffer.width, bitmapBuffer.height, matrix, true - ) - image_predicted.setImageBitmap(uprightImage) + bitmapBuffer?.let { + val uprightImage = Bitmap.createBitmap(it, 0, 0, it.width, it.height, matrix, true) + image_predicted.setImageBitmap(uprightImage) + } image_predicted.visibility = View.VISIBLE } @@ -176,25 +181,21 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { // camera framerate. Log.w("sessionUpdate", Thread.currentThread().name) - val frame: Frame = session.update() - val image: Image - try { - frame.acquireCameraImage().use { image -> - if (!::bitmapBuffer.isInitialized) { + surfaceView.queueEvent { + val frame: Frame = session.update() + try { + frame.acquireCameraImage().use { image -> // The image rotation and RGB image buffer are initialized only once // the analyzer has started running imageRotationDegrees = 0 - bitmapBuffer = Bitmap.createBitmap( - image.width, image.height, Bitmap.Config.ARGB_8888 - ) + bitmapBuffer = Bitmap.createBitmap(image.width, image.height, Bitmap.Config.ARGB_8888) } + } catch (e: NotYetAvailableException) { + // This normally means that depth data is not available yet. This is normal so we will not + // spam the logcat with this. } - } catch (e: NotYetAvailableException) { - // This normally means that depth data is not available yet. This is normal so we will not - // spam the logcat with this. } - // Set up the image analysis use case which will process frames in real time /*val imageAnalysis = ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) @@ -209,55 +210,59 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { //image.use { converter.yuvToRgb(image, bitmapBuffer) } HOW to convert from yuv to rgb // Process the image in Tensorflow - val tfImage = tfImageProcessor.process(tfImageBuffer.apply { load(bitmapBuffer) }) + bitmapBuffer?.let { bitmap -> + val tfImage = tfImageProcessor?.process(tfImageBuffer.apply { load(bitmap) }) - // Perform the object detection for the current frame - val predictions = detector.predict(tfImage) - // Report only the top prediction - surfaceView.queueEvent { reportPrediction(predictions.maxByOrNull { it.score }) } + tfImage?.let { + // Perform the object detection for the current frame + val predictions = detector.predict(it) - /*imageAnalysis.setAnalyzer(executor, ImageAnalysis.Analyzer { image -> - if (!::bitmapBuffer.isInitialized) { - // The image rotation and RGB image buffer are initialized only once - // the analyzer has started running - imageRotationDegrees = image.imageInfo.rotationDegrees - bitmapBuffer = Bitmap.createBitmap( - image.width, image.height, Bitmap.Config.ARGB_8888 - ) - } + // Report only the top prediction + surfaceView.queueEvent { reportPrediction(predictions.maxByOrNull { it.score }) } - // Early exit: image analysis is in paused state - if (pauseAnalysis) { - image.close() - return@Analyzer - } + /*imageAnalysis.setAnalyzer(executor, ImageAnalysis.Analyzer { image -> + if (!::bitmapBuffer.isInitialized) { + // The image rotation and RGB image buffer are initialized only once + // the analyzer has started running + imageRotationDegrees = image.imageInfo.rotationDegrees + bitmapBuffer = Bitmap.createBitmap( + image.width, image.height, Bitmap.Config.ARGB_8888 + ) + } - // Convert the image to RGB and place it in our shared buffer - image.use { converter.yuvToRgb(image.image!!, bitmapBuffer) } + // Early exit: image analysis is in paused state + if (pauseAnalysis) { + image.close() + return@Analyzer + } - // Process the image in Tensorflow - val tfImage = tfImageProcessor.process(tfImageBuffer.apply { load(bitmapBuffer) }) + // Convert the image to RGB and place it in our shared buffer + image.use { converter.yuvToRgb(image.image!!, bitmapBuffer) } - // Perform the object detection for the current frame - val predictions = detector.predict(tfImage) + // Process the image in Tensorflow + val tfImage = tfImageProcessor.process(tfImageBuffer.apply { load(bitmapBuffer) }) - // Report only the top prediction - reportPrediction(predictions.maxBy { it.score }) + // Perform the object detection for the current frame + val predictions = detector.predict(tfImage) - // Compute the FPS of the entire pipeline - val frameCount = 10 - if (++frameCounter % frameCount == 0) { - frameCounter = 0 - val now = System.currentTimeMillis() - val delta = now - lastFpsTimestamp - val fps = 1000 * frameCount.toFloat() / delta - Log.d(TAG, "FPS: ${"%.02f".format(fps)}") - lastFpsTimestamp = now - } - }) - */ + // Report only the top prediction + reportPrediction(predictions.maxBy { it.score }) + // Compute the FPS of the entire pipeline + val frameCount = 10 + if (++frameCounter % frameCount == 0) { + frameCounter = 0 + val now = System.currentTimeMillis() + val delta = now - lastFpsTimestamp + val fps = 1000 * frameCount.toFloat() / delta + Log.d(TAG, "FPS: ${"%.02f".format(fps)}") + lastFpsTimestamp = now + } + }) + */ + } + } // Create a new camera selector each time, enforcing lens facing val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build() From 06f97b41bbdfb9e7cbee3885fba214789d8ad11c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 22 Mar 2021 23:30:12 -0400 Subject: [PATCH 3/4] Setting camera texture --- .../com/android/example/camerax/tflite/CameraActivity.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt index b0ccb38..c1ffd0c 100644 --- a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt +++ b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt @@ -182,6 +182,8 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { Log.w("sessionUpdate", Thread.currentThread().name) surfaceView.queueEvent { + //session.setCameraTextureName(0) + session!!.setCameraTextureNames(intArrayOf(0)) val frame: Frame = session.update() try { frame.acquireCameraImage().use { image -> @@ -382,6 +384,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { override fun onPause() { super.onPause() surfaceView.onPause() + GLES20.glGenTextures(1, IntArray(1), 0) session.pause() isFirstFrameAfterResume.set(true) } @@ -423,6 +426,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { override fun onSurfaceCreated(p0: GL10?, p1: EGLConfig?) { GLES20.glClearColor(0.1f, 0.1f, 0.1f, 1.0f) + } override fun onSurfaceChanged(p0: GL10?, p1: Int, p2: Int) { @@ -431,7 +435,7 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { override fun onDrawFrame(p0: GL10?) { Log.w("sessionDraw", Thread.currentThread().name) - + session!!.setCameraTextureNames(intArrayOf(0)) val frame = session.update() // Place an object on tap. @@ -459,3 +463,4 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { } } + From 5ad72fc32ab6b718d264e8660cca944c89f97676 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 22 Mar 2021 23:30:12 -0400 Subject: [PATCH 4/4] Setting camera texture --- .../java/com/android/example/camerax/tflite/CameraActivity.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt index c1ffd0c..9cf7fc7 100644 --- a/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt +++ b/app/src/main/java/com/android/example/camerax/tflite/CameraActivity.kt @@ -445,6 +445,8 @@ class CameraActivity : AppCompatActivity(), GLSurfaceView.Renderer { val approximateDistanceMeters = 2.0f // Performs a ray cast given a screen tap position. val results = frame.hitTestInstantPlacement(0F, 0F, approximateDistanceMeters) + val displayRotation = display!!.rotation + session.setDisplayGeometry(displayRotation, 256, 256) if (results.isNotEmpty()) { val point = results[0].trackable as InstantPlacementPoint // Create an Anchor from the point's pose.