Skip to content

SDK / exoplayer

Go to a project
Commit e25bf811 by hschlueter Committed by Ian Baker
Options

Use GlTextureProcessor to avoid redundant copy in MediaPipeProcessor. 

After this change GlEffects can use any GlTextureProcessor not just
SingleFrameGlTextureProcessor.
MediaPipeProcessor now implements GlTextureProcessor directly which
allows it to reuse MediaPipe's output texture for its output texture
and avoids an extra copy shader step.

PiperOrigin-RevId: 456530718
parent 3a966916
Inline Side-by-side
Showing with 92 additions and 86 deletions
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java
......@@ -41,8 +41,8 @@ import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.transformer.DefaultEncoderFactory;
import com.google.android.exoplayer2.transformer.EncoderSelector;
import com.google.android.exoplayer2.transformer.GlEffect;
import com.google.android.exoplayer2.transformer.GlTextureProcessor;
import com.google.android.exoplayer2.transformer.ProgressHolder;
import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor;
import com.google.android.exoplayer2.transformer.TransformationException;
import com.google.android.exoplayer2.transformer.TransformationRequest;
import com.google.android.exoplayer2.transformer.TransformationResult;
......@@ -282,7 +282,7 @@ public final class TransformerActivity extends AppCompatActivity {
effects.add(
(Context context) -> {
try {
return (SingleFrameGlTextureProcessor)
return (GlTextureProcessor)
constructor.newInstance(
context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb",
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
......@@ -20,27 +20,24 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.EGL14;
import android.opengl.GLES20;
import android.util.Size;
import android.os.Build;
import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.transformer.FrameProcessingException;
import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor;
import com.google.android.exoplayer2.util.ConditionVariable;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.transformer.GlTextureProcessor;
import com.google.android.exoplayer2.transformer.TextureInfo;
import com.google.android.exoplayer2.util.LibraryLoader;
import com.google.android.exoplayer2.util.Util;
import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.framework.AppTextureFrame;
import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.EglManager;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that
* can immediately produce one output frame per input frame.
*/
/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor {
/** Runs a MediaPipe graph on input frames. */
/* package */ final class MediaPipeProcessor implements GlTextureProcessor {
private static final LibraryLoader LOADER =
new LibraryLoader("mediapipe_jni") {
......@@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";
private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor;
private final GlProgram glProgram;
private int inputWidth;
private int inputHeight;
private @MonotonicNonNull TextureFrame outputFrame;
private @MonotonicNonNull RuntimeException frameProcessorPendingError;
private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
private volatile boolean acceptedFrame;
// Only available from API 23 and above.
@Nullable private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
// Used instead for API 21 and 22.
@Nullable private volatile TextureInfo outputTexture;
@Nullable private volatile TextureFrame outputFrame;
/**
* Creates a new texture processor that wraps a MediaPipe graph.
......@@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph.
* @throws FrameProcessingException If a problem occurs while reading shader files or initializing
* MediaPipe resources.
*/
@SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23.
public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName)
throws FrameProcessingException {
Context context, String graphName, String inputStreamName, String outputStreamName) {
checkState(LOADER.isAvailable());
frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context);
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor =
new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
// Unblock drawFrame when there is an output frame or an error.
outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null;
frameProcessor.setConsumer(
frame -> {
outputFrame = frame;
frameProcessorConditionVariable.open();
TextureInfo texture =
new TextureInfo(
frame.getTextureName(),
/* fboId= */ C.INDEX_UNSET,
frame.getWidth(),
frame.getHeight());
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(outputFrames).put(texture, frame);
} else {
outputFrame = frame;
outputTexture = texture;
}
if (listener != null) {
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
}
});
frameProcessor.setAsynchronousErrorListener(
error -> {
frameProcessorPendingError = error;
frameProcessorConditionVariable.open();
if (listener != null) {
listener.onFrameProcessingError(new FrameProcessingException(error));
}
});
try {
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true);
}
@Override
public Size configure(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight);
public void setListener(GlTextureProcessor.Listener listener) {
this.listener = listener;
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close();
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (!areMultipleOutputFramesSupported() && outputTexture != null) {
return false;
}
// Pass the input frame to MediaPipe.
AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight);
acceptedFrame = false;
AppTextureFrame appTextureFrame =
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
appTextureFrame.setTimestamp(presentationTimeUs);
checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame);
// Wait for output to be passed to the consumer.
try {
frameProcessorConditionVariable.block();
appTextureFrame.waitUntilReleasedWithGpuSync();
} catch (InterruptedException e) {
// Propagate the interrupted flag so the next blocking operation will throw.
// TODO(b/230469581): The next processor that runs will not have valid input due to returning
// early here. This could be fixed by checking for interruption in the outer loop that runs
// through the texture processors.
Thread.currentThread().interrupt();
return;
if (listener != null) {
listener.onFrameProcessingError(new FrameProcessingException(e));
}
}
if (frameProcessorPendingError != null) {
throw new FrameProcessingException(frameProcessorPendingError);
if (listener != null) {
listener.onInputFrameProcessed(inputTexture);
}
return acceptedFrame;
}
// Copy from MediaPipe's output texture to the current output.
try {
checkStateNotNull(glProgram).use();
glProgram.setSamplerTexIdUniform(
"uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0);
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
} finally {
@Override
public void releaseOutputFrame(TextureInfo outputTexture) {
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release();
} else {
checkState(Util.areEqual(outputTexture, this.outputTexture));
this.outputTexture = null;
checkStateNotNull(outputFrame).release();
outputFrame = null;
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
public void release() {
checkStateNotNull(frameProcessor).close();
}
@Override
public final void signalEndOfInputStream() {
frameProcessor.waitUntilIdle();
if (listener != null) {
listener.onOutputStreamEnded();
}
}
@ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M)
private static boolean areMultipleOutputFramesSupported() {
// Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result
// in lost updates, so we only allow one output frame to be pending at a time to avoid using
// ConcurrentHashMap.
return Util.SDK_INT >= 23;
}
}
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java
......@@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest {
}
@Override
public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context)
public GlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException {
return effect.toGlTextureProcessor(context);
}
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
......@@ -23,6 +23,7 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceHolder;
......@@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
renderingTask.run();
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
GLES20.glFinish();
}
@Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java
......@@ -18,16 +18,14 @@ package com.google.android.exoplayer2.transformer;
import android.content.Context;
/**
* Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation.
* Interface for a video frame effect with a {@link GlTextureProcessor} implementation.
*
* <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which
* applies the effect.
* #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the
* effect.
*/
public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */
// TODO(b/227625423): use GlTextureProcessor here once this interface exists.
SingleFrameGlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException;
GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException;
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java
......@@ -16,7 +16,7 @@
package com.google.android.exoplayer2.transformer;
/** Contains information describing an OpenGL texture. */
/* package */ final class TextureInfo {
public final class TextureInfo {
/** The OpenGL texture identifier. */
public final int texId;
/** Identifier of a framebuffer object associated with the texture. */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment