Skip to content

SDK / exoplayer

Go to a project
Commit 6edd1d2a by hschlueter Committed by microkatz
Options

Use GlTextureProcessor to avoid redundant copy in MediaPipeProcessor. 

After this change GlEffects can use any GlTextureProcessor not just
SingleFrameGlTextureProcessor.
MediaPipeProcessor now implements GlTextureProcessor directly which
allows it to reuse MediaPipe's output texture for its output texture
and avoids an extra copy shader step.

PiperOrigin-RevId: 456530718
(cherry picked from commit e25bf811)
parent f96a6c71
Inline Side-by-side
Showing with 91 additions and 85 deletions
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java
...@@ -41,8 +41,8 @@ import com.google.android.exoplayer2.MediaItem; ...@@ -41,8 +41,8 @@ import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory;
import com.google.android.exoplayer2.transformer.EncoderSelector; import com.google.android.exoplayer2.transformer.EncoderSelector;
import com.google.android.exoplayer2.transformer.GlEffect; import com.google.android.exoplayer2.transformer.GlEffect;
import com.google.android.exoplayer2.transformer.GlTextureProcessor;
import com.google.android.exoplayer2.transformer.ProgressHolder; import com.google.android.exoplayer2.transformer.ProgressHolder;
import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor;
import com.google.android.exoplayer2.transformer.TransformationException; import com.google.android.exoplayer2.transformer.TransformationException;
import com.google.android.exoplayer2.transformer.TransformationRequest; import com.google.android.exoplayer2.transformer.TransformationRequest;
import com.google.android.exoplayer2.transformer.TransformationResult; import com.google.android.exoplayer2.transformer.TransformationResult;
...@@ -282,7 +282,7 @@ public final class TransformerActivity extends AppCompatActivity { ...@@ -282,7 +282,7 @@ public final class TransformerActivity extends AppCompatActivity {
effects.add( effects.add(
(Context context) -> { (Context context) -> {
try { try {
return (SingleFrameGlTextureProcessor) return (GlTextureProcessor)
constructor.newInstance( constructor.newInstance(
context, context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* graphName= */ "edge_detector_mediapipe_graph.binarypb",
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
...@@ -20,27 +20,24 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; ...@@ -20,27 +20,24 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.GLES20; import android.os.Build;
import android.util.Size; import androidx.annotation.ChecksSdkIntAtLeast;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.transformer.FrameProcessingException; import com.google.android.exoplayer2.transformer.FrameProcessingException;
import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor; import com.google.android.exoplayer2.transformer.GlTextureProcessor;
import com.google.android.exoplayer2.util.ConditionVariable; import com.google.android.exoplayer2.transformer.TextureInfo;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.LibraryLoader; import com.google.android.exoplayer2.util.LibraryLoader;
import com.google.android.exoplayer2.util.Util;
import com.google.mediapipe.components.FrameProcessor; import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.framework.AppTextureFrame; import com.google.mediapipe.framework.AppTextureFrame;
import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.EglManager; import com.google.mediapipe.glutil.EglManager;
import java.io.IOException; import java.util.concurrent.ConcurrentHashMap;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /** Runs a MediaPipe graph on input frames. */
* Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that /* package */ final class MediaPipeProcessor implements GlTextureProcessor {
* can immediately produce one output frame per input frame.
*/
/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor {
private static final LibraryLoader LOADER = private static final LibraryLoader LOADER =
new LibraryLoader("mediapipe_jni") { new LibraryLoader("mediapipe_jni") {
...@@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";
private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor; private final FrameProcessor frameProcessor;
private final GlProgram glProgram; private volatile GlTextureProcessor.@MonotonicNonNull Listener listener;
private volatile boolean acceptedFrame;
private int inputWidth; // Only available from API 23 and above.
private int inputHeight; @Nullable private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
private @MonotonicNonNull TextureFrame outputFrame; // Used instead for API 21 and 22.
private @MonotonicNonNull RuntimeException frameProcessorPendingError; @Nullable private volatile TextureInfo outputTexture;
@Nullable private volatile TextureFrame outputFrame;
/** /**
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
...@@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param graphName Name of a MediaPipe graph asset to load. * @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph. * @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph.
* @throws FrameProcessingException If a problem occurs while reading shader files or initializing
* MediaPipe resources.
*/ */
@SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23.
public MediaPipeProcessor( public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName) Context context, String graphName, String inputStreamName, String outputStreamName) {
throws FrameProcessingException {
checkState(LOADER.isAvailable()); checkState(LOADER.isAvailable());
frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context);
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
// Unblock drawFrame when there is an output frame or an error. outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null;
frameProcessor.setConsumer( frameProcessor.setConsumer(
frame -> { frame -> {
TextureInfo texture =
new TextureInfo(
frame.getTextureName(),
/* fboId= */ C.INDEX_UNSET,
frame.getWidth(),
frame.getHeight());
if (areMultipleOutputFramesSupported()) {
checkStateNotNull(outputFrames).put(texture, frame);
} else {
outputFrame = frame; outputFrame = frame;
frameProcessorConditionVariable.open(); outputTexture = texture;
}
if (listener != null) {
listener.onOutputFrameAvailable(texture, frame.getTimestamp());
}
}); });
frameProcessor.setAsynchronousErrorListener( frameProcessor.setAsynchronousErrorListener(
error -> { error -> {
frameProcessorPendingError = error; if (listener != null) {
frameProcessorConditionVariable.open(); listener.onFrameProcessingError(new FrameProcessingException(error));
});
try {
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
} }
});
frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true);
} }
@Override @Override
public Size configure(int inputWidth, int inputHeight) { public void setListener(GlTextureProcessor.Listener listener) {
this.inputWidth = inputWidth; this.listener = listener;
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight);
} }
@Override @Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
frameProcessorConditionVariable.close(); if (!areMultipleOutputFramesSupported() && outputTexture != null) {
return false;
}
// Pass the input frame to MediaPipe. acceptedFrame = false;
AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight); AppTextureFrame appTextureFrame =
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
appTextureFrame.setTimestamp(presentationTimeUs); appTextureFrame.setTimestamp(presentationTimeUs);
checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame); checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame);
// Wait for output to be passed to the consumer.
try { try {
frameProcessorConditionVariable.block(); appTextureFrame.waitUntilReleasedWithGpuSync();
} catch (InterruptedException e) { } catch (InterruptedException e) {
// Propagate the interrupted flag so the next blocking operation will throw.
// TODO(b/230469581): The next processor that runs will not have valid input due to returning
// early here. This could be fixed by checking for interruption in the outer loop that runs
// through the texture processors.
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
return; if (listener != null) {
listener.onFrameProcessingError(new FrameProcessingException(e));
} }
}
if (frameProcessorPendingError != null) { if (listener != null) {
throw new FrameProcessingException(frameProcessorPendingError); listener.onInputFrameProcessed(inputTexture);
}
return acceptedFrame;
} }
// Copy from MediaPipe's output texture to the current output. @Override
try { public void releaseOutputFrame(TextureInfo outputTexture) {
checkStateNotNull(glProgram).use(); if (areMultipleOutputFramesSupported()) {
glProgram.setSamplerTexIdUniform( checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release();
"uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0); } else {
glProgram.setBufferAttribute( checkState(Util.areEqual(outputTexture, this.outputTexture));
"aFramePosition", this.outputTexture = null;
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
} finally {
checkStateNotNull(outputFrame).release(); checkStateNotNull(outputFrame).release();
outputFrame = null;
} }
} }
@Override @Override
public void release() throws FrameProcessingException { public void release() {
super.release();
checkStateNotNull(frameProcessor).close(); checkStateNotNull(frameProcessor).close();
} }
@Override
public final void signalEndOfInputStream() {
frameProcessor.waitUntilIdle();
if (listener != null) {
listener.onOutputStreamEnded();
}
}
@ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M)
private static boolean areMultipleOutputFramesSupported() {
// Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result
// in lost updates, so we only allow one output frame to be pending at a time to avoid using
// ConcurrentHashMap.
return Util.SDK_INT >= 23;
}
} }
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java
...@@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest { ...@@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest {
} }
@Override @Override
public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) public GlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException { throws FrameProcessingException {
return effect.toGlTextureProcessor(context); return effect.toGlTextureProcessor(context);
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
...@@ -23,6 +23,7 @@ import android.opengl.EGLContext; ...@@ -23,6 +23,7 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay; import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceHolder; import android.view.SurfaceHolder;
...@@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
renderingTask.run(); renderingTask.run();
EGL14.eglSwapBuffers(eglDisplay, eglSurface); EGL14.eglSwapBuffers(eglDisplay, eglSurface);
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
GLES20.glFinish();
} }
@Override @Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java
...@@ -18,16 +18,14 @@ package com.google.android.exoplayer2.transformer; ...@@ -18,16 +18,14 @@ package com.google.android.exoplayer2.transformer;
import android.content.Context; import android.content.Context;
/** /**
* Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. * Interface for a video frame effect with a {@link GlTextureProcessor} implementation.
* *
* <p>Implementations contain information specifying the effect and can be {@linkplain * <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which * #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the
* applies the effect. * effect.
*/ */
public interface GlEffect { public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */
// TODO(b/227625423): use GlTextureProcessor here once this interface exists. GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException;
SingleFrameGlTextureProcessor toGlTextureProcessor(Context context)
throws FrameProcessingException;
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
/** Contains information describing an OpenGL texture. */ /** Contains information describing an OpenGL texture. */
/* package */ final class TextureInfo { public final class TextureInfo {
/** The OpenGL texture identifier. */ /** The OpenGL texture identifier. */
public final int texId; public final int texId;
/** Identifier of a framebuffer object associated with the texture. */ /** Identifier of a framebuffer object associated with the texture. */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment