Hello StackOverflow community.
I’m currently building a Pose Estimation Android App which detects keypoints of human body joints. I’m using Movenet’s Singlepose Lighting model and was trying to integrate into Android Application.
Here’s my issue.
Link to the Model
The model seems to not accept the image and always returning E Unsupported image format: 1 into the Logcat.
Snippet of my code:
CameraActivity (startCamera function)
private fun startCamera() {
val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
cameraProviderFuture.addListener({
val resolutionSelector = ResolutionSelector.Builder()
.setAspectRatioStrategy(AspectRatioStrategy.RATIO_16_9_FALLBACK_AUTO_STRATEGY)
.build()
val imageAnalyzer = ImageAnalysis.Builder()
.setResolutionSelector(resolutionSelector)
.setTargetRotation(binding.viewFinder.display.rotation)
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build()
imageAnalyzer.setAnalyzer(Executors.newSingleThreadExecutor()) { image ->
poseEstimationHelper.detectPose(image)
}
val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
val preview = Preview.Builder().build().also {
it.setSurfaceProvider(binding.viewFinder.surfaceProvider)
}
try {
cameraProvider.unbindAll()
cameraProvider.bindToLifecycle(
this,
cameraSelector,
preview,
imageAnalyzer
)
} catch (exc: Exception) {
Toast.makeText(
this@CameraActivity,
"Failed to start the camera.",
Toast.LENGTH_SHORT
).show()
Log.e(TAG, "startCamera: ${exc.message}")
}
}, ContextCompat.getMainExecutor(this))
}
PoseEstimationHelper
class PoseEstimationHelper(
val context: Context,
val detectorListener: DetectorListener?,
private val onError: (String) -> Unit,
) {
private var interpreter: Interpreter? = null
private var initializationTask: Task<Void>? = null
init {
initializationTask = TfLiteGpu.isGpuDelegateAvailable(context).onSuccessTask { gpuAvailable ->
val optionsBuilder = TfLiteInitializationOptions.builder()
if (gpuAvailable) {
optionsBuilder.setEnableGpuDelegateSupport(true)
}
TfLiteVision.initialize(context, optionsBuilder.build())
}.addOnSuccessListener {
Log.d(TAG, "TfLiteVision initialized successfully")
downloadModel()
}.addOnFailureListener {
val errorMsg = context.getString(R.string.tflitevision_is_not_initialized_yet)
detectorListener?.onError(errorMsg)
onError(errorMsg)
Log.e(TAG, "TfLiteVision initialization failed", it)
}
}
@Synchronized
private fun downloadModel() {
val conditions = CustomModelDownloadConditions.Builder()
.requireWifi()
.build()
FirebaseModelDownloader.getInstance()
.getModel("singlepose-movenet-lightning", DownloadType.LOCAL_MODEL, conditions)
.addOnSuccessListener { model: CustomModel ->
try {
val modelFile = model.file
if (modelFile != null) {
setupModel(modelFile)
} else {
throw IOException("Model file is null")
}
} catch (e: IOException) {
val errorMsg = e.message.toString()
detectorListener?.onError(errorMsg)
onError(errorMsg)
}
}
.addOnFailureListener { e: Exception? ->
val errorMsg = context.getString(R.string.firebaseml_model_download_failed)
detectorListener?.onError(errorMsg)
onError(errorMsg)
Log.e(TAG, "Model download failed", e)
}
}
private fun setupModel(modelFile: File) {
val options = Interpreter.Options()
if (CompatibilityList().isDelegateSupportedOnThisDevice) {
options.addDelegate(org.tensorflow.lite.gpu.GpuDelegate())
} else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
options.useNNAPI = true
} else {
options.setNumThreads(4)
}
try {
interpreter = Interpreter(modelFile, options)
Log.d(TAG, "Pose estimation model set up successfully")
} catch (e: IOException) {
val errorMsg = context.getString(R.string.image_classifier_failed)
detectorListener?.onError(errorMsg)
onError(errorMsg)
Log.e(TAG, "Pose estimation model setup failed", e)
}
}
fun detectPose(image: ImageProxy) {
initializationTask?.addOnSuccessListener {
if (!TfLiteVision.isInitialized()) {
val errorMessage = context.getString(R.string.tflitevision_is_not_initialized_yet)
Log.e(TAG, errorMessage)
detectorListener?.onError(errorMessage)
onError(errorMessage)
return@addOnSuccessListener
}
if (interpreter == null) {
val errorMsg = "Pose estimation model is not set up yet"
detectorListener?.onError(errorMsg)
onError(errorMsg)
return@addOnSuccessListener
}
try {
val bitmap = toBitmap(image)
val resizedBitmap = Bitmap.createScaledBitmap(bitmap, 192, 192, true)
val imageProcessor = ImageProcessor.Builder()
.add(ResizeOp(192, 192, ResizeOp.ResizeMethod.BILINEAR))
.add(Rot90Op(-image.imageInfo.rotationDegrees / 90))
.add(NormalizeOp(0f, 1f))
.build()
val tensorImage = imageProcessor.process(TensorImage.fromBitmap(resizedBitmap))
val inputFeature0 = TensorBuffer.createFixedSize(intArrayOf(1, 192, 192, 3), DataType.FLOAT32)
inputFeature0.loadBuffer(tensorImage.buffer)
val outputBuffer = TensorBuffer.createFixedSize(intArrayOf(1, 1, 17, 3), DataType.FLOAT32)
var inferenceTime = SystemClock.uptimeMillis()
interpreter?.run(inputFeature0.buffer, outputBuffer.buffer.rewind())
inferenceTime = SystemClock.uptimeMillis() - inferenceTime
val keypoints = extractKeypoints(outputBuffer)
detectorListener?.onResults(keypoints, inferenceTime)
} catch (e: IllegalArgumentException) {
val errorMessage = e.message ?: "Unknown error"
Log.e(TAG, errorMessage)
detectorListener?.onError(errorMessage)
onError(errorMessage)
} finally {
image.close()
}
}?.addOnFailureListener {
val errorMessage = context.getString(R.string.tflitevision_is_not_initialized_yet)
Log.e(TAG, errorMessage)
detectorListener?.onError(errorMessage)
onError(errorMessage)
}
}
private fun toBitmap(image: ImageProxy): Bitmap {
Log.d(TAG, "Image format: ${image.format}")
return when (image.format) {
ImageFormat.YUV_420_888 -> {
val yBuffer = image.planes[0].buffer
val uBuffer = image.planes[1].buffer
val vBuffer = image.planes[2].buffer
val ySize = yBuffer.remaining()
val uSize = uBuffer.remaining()
val vSize = vBuffer.remaining()
val nv21 = ByteArray(ySize + uSize + vSize)
yBuffer.get(nv21, 0, ySize)
vBuffer.get(nv21, ySize, vSize)
uBuffer.get(nv21, ySize + vSize, uSize)
val yuvImage = YuvImage(nv21, ImageFormat.NV21, image.width, image.height, null)
val out = ByteArrayOutputStream()
yuvImage.compressToJpeg(Rect(0, 0, image.width, image.height), 100, out)
val yuvByteArray = out.toByteArray()
BitmapFactory.decodeByteArray(yuvByteArray, 0, yuvByteArray.size)
}
ImageFormat.JPEG -> {
val buffer = image.planes[0].buffer
Log.d(TAG, "JPEG buffer size: ${buffer.remaining()}")
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
ImageFormat.RGB_565 -> {
val buffer = image.planes[0].buffer
val bytes = ByteArray(buffer.remaining())
buffer.get(bytes)
BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
}
else -> {
val errorMsg = "Unsupported image format: ${image.format}"
Log.e(TAG, errorMsg)
throw IllegalArgumentException(errorMsg)
}
}
}
private fun extractKeypoints(outputBuffer: TensorBuffer): List<Keypoint> {
val keypoints = mutableListOf<Keypoint>()
val scores = outputBuffer.floatArray
// Assume that each keypoint has 3 values: x, y, and score
for (i in scores.indices step 3) {
val x = scores[i]
val y = scores[i + 1]
val score = scores[i + 2]
keypoints.add(Keypoint(x, y, score))
}
return keypoints
}
interface DetectorListener {
fun onError(error: String)
fun onResults(
keypoints: List<Keypoint>?,
inferenceTime: Long
)
}
data class Keypoint(val x: Float, val y: Float, val score: Float)
companion object {
private const val TAG = "PoseEstimationHelper"
}
}
Documentation in the internet is not well enough to establish a clear understanding of what’s really happening.
Any help is greatly appreciated.
Thank you very much.
I have tried to change the toBitmap function, adding numerous different ImageFormat scenario. Still returns the same error.
New contributor
SpaceShuttle is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.