Skip to content

SDK / exoplayer

Go to a project
Commit 719715d2 by huangdarwin Committed by Tofunmi Adigun-Hameed
Options

Effect: Multiple Texture output 

Allow the VideoFrameProcessor to output multiple textures at a time, so that
lifetime of textures is up to the consumer calling VFP.releaseOutputFrame.

The FinalShaderProgramWrapper also has a new maxCapacity limit added, to ensure
the a reasonable amount of textures is used and avoid using up memory.

PiperOrigin-RevId: 532094256
(cherry picked from commit 025f71bec45e06b19b7237967af2677b84387b08)
parent 568233f6
Expand all
Inline Side-by-side
Showing with 188 additions and 62 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
...@@ -143,7 +143,7 @@ public interface VideoFrameProcessor { ...@@ -143,7 +143,7 @@ public interface VideoFrameProcessor {
*/ */
void onError(VideoFrameProcessingException exception); void onError(VideoFrameProcessingException exception);
/** Called after the {@link VideoFrameProcessor} has produced its final output frame. */ /** Called after the {@link VideoFrameProcessor} has rendered its final output frame. */
void onEnded(); void onEnded();
} }
...@@ -290,6 +290,18 @@ public interface VideoFrameProcessor { ...@@ -290,6 +290,18 @@ public interface VideoFrameProcessor {
void renderOutputFrame(long renderTimeNs); void renderOutputFrame(long renderTimeNs);
/** /**
* Releases resources associated with all output frames with presentation time less than or equal
* to {@code presentationTimeUs}.
*
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
* be required for other outputs.
*
* @param presentationTimeUs The presentation time where all frames before and at this time should
* be released, in microseconds.
*/
void releaseOutputFrame(long presentationTimeUs);
/**
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted. * Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
* *
* <p>Can be called on any thread. * <p>Can be called on any thread.
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
...@@ -31,6 +31,7 @@ import android.opengl.GLES20; ...@@ -31,6 +31,7 @@ import android.opengl.GLES20;
import android.opengl.GLES30; import android.opengl.GLES30;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.GuardedBy; import androidx.annotation.GuardedBy;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
...@@ -71,7 +72,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -71,7 +72,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public interface TextureOutputListener { public interface TextureOutputListener {
/** Called when a texture has been rendered to. */ /** Called when a texture has been rendered to. */
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs) void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException; throws VideoFrameProcessingException;
} }
/** A factory for {@link DefaultVideoFrameProcessor} instances. */ /** A factory for {@link DefaultVideoFrameProcessor} instances. */
...@@ -82,6 +83,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -82,6 +83,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private boolean enableColorTransfers; private boolean enableColorTransfers;
private GlObjectsProvider glObjectsProvider; private GlObjectsProvider glObjectsProvider;
@Nullable private TextureOutputListener textureOutputListener; @Nullable private TextureOutputListener textureOutputListener;
private int textureOutputCapacity;
/** Creates an instance. */ /** Creates an instance. */
public Builder() { public Builder() {
...@@ -112,39 +114,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -112,39 +114,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
} }
/** /**
* Sets the {@link TextureOutputListener}. * Sets texture output settings.
* *
* <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via * <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
* {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to. * {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
* {@code textureOutputCapacity} is reached, until they're released via {@link
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
* *
* <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will * <p>If not set, there will be no texture output.
* be be adjusted as needed, to match the output surface's output. *
* <p>This must not be set if the {@linkplain #setOutputSurfaceInfo output surface info} is
* also set.
*
* @param textureOutputListener The {@link TextureOutputListener}.
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
* before texture output blocks. Must be greater than or equal to 1.
*/ */
@VisibleForTesting @VisibleForTesting
@CanIgnoreReturnValue @CanIgnoreReturnValue
public Builder setOnTextureRenderedListener(TextureOutputListener textureOutputListener) { public Builder setTextureOutput(
TextureOutputListener textureOutputListener,
@IntRange(from = 1) int textureOutputCapacity) {
// TODO: http://b/262694346 - Add tests for multiple texture output.
this.textureOutputListener = textureOutputListener; this.textureOutputListener = textureOutputListener;
checkArgument(textureOutputCapacity >= 1);
this.textureOutputCapacity = textureOutputCapacity;
return this; return this;
} }
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */ /** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
public DefaultVideoFrameProcessor.Factory build() { public DefaultVideoFrameProcessor.Factory build() {
return new DefaultVideoFrameProcessor.Factory( return new DefaultVideoFrameProcessor.Factory(
enableColorTransfers, glObjectsProvider, textureOutputListener); enableColorTransfers, glObjectsProvider, textureOutputListener, textureOutputCapacity);
} }
} }
private final boolean enableColorTransfers; private final boolean enableColorTransfers;
private final GlObjectsProvider glObjectsProvider; private final GlObjectsProvider glObjectsProvider;
@Nullable private final TextureOutputListener textureOutputListener; @Nullable private final TextureOutputListener textureOutputListener;
private final int textureOutputCapacity;
private Factory( private Factory(
boolean enableColorTransfers, boolean enableColorTransfers,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) { @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity) {
this.enableColorTransfers = enableColorTransfers; this.enableColorTransfers = enableColorTransfers;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener; this.textureOutputListener = textureOutputListener;
this.textureOutputCapacity = textureOutputCapacity;
} }
/** /**
...@@ -229,7 +247,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -229,7 +247,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
listenerExecutor, listenerExecutor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener)); textureOutputListener,
textureOutputCapacity));
try { try {
return defaultVideoFrameProcessorFuture.get(); return defaultVideoFrameProcessorFuture.get();
...@@ -409,11 +428,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -409,11 +428,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return checkNotNull(textureManager).getPendingFrameCount(); return checkNotNull(textureManager).getPendingFrameCount();
} }
/**
* {@inheritDoc}
*
* <p>This must not be set on an instance where {@linkplain Factory.Builder#setTextureOutput
* texture output} is set.
*/
@Override @Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo); finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
} }
/**
* {@inheritDoc}
*
* <p>This may also be used for rendering from an output texture, if a {@link
* TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set}
*/
@Override @Override
public void renderOutputFrame(long renderTimeNs) { public void renderOutputFrame(long renderTimeNs) {
checkState( checkState(
...@@ -423,6 +454,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -423,6 +454,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs)); () -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
} }
/**
* {@inheritDoc}
*
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
* this must be called to release the output information stored in the {@link GlTextureInfo}
* instances.
*/
@Override
public void releaseOutputFrame(long presentationTimeUs) {
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
// input is possible.
videoFrameProcessingTaskExecutor.submit(
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
}
@Override @Override
public void signalEndOfInput() { public void signalEndOfInput() {
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos(); DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
...@@ -509,7 +555,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -509,7 +555,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor, Executor executor,
Listener listener, Listener listener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws GlUtil.GlException, VideoFrameProcessingException { throws GlUtil.GlException, VideoFrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
...@@ -566,7 +613,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -566,7 +613,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener); textureOutputListener,
textureOutputCapacity);
inputSwitcher.registerInput(INPUT_TYPE_SURFACE); inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
if (!ColorInfo.isTransferHdr(inputColorInfo)) { if (!ColorInfo.isTransferHdr(inputColorInfo)) {
...@@ -616,7 +664,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -616,7 +664,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor, Executor executor,
Listener listener, Listener listener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder = ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
...@@ -668,7 +717,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -668,7 +717,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener)); textureOutputListener,
textureOutputCapacity));
return shaderProgramListBuilder.build(); return shaderProgramListBuilder.build();
} }
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
...@@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener( .setTextureOutput(
(outputTexture, presentationTimeUs) -> (outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor( inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs)) testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build(); .build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder() new VideoFrameProcessorTestRunner.Builder()
...@@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener( .setTextureOutput(
(outputTexture, presentationTimeUs) -> (outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor( inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs)) testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build(); .build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder() new VideoFrameProcessorTestRunner.Builder()
...@@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
new DefaultGlObjectsProvider(GlUtil.getCurrentContext()); new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(bitmapReader::readBitmapFromTexture) .setTextureOutput(bitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.setGlObjectsProvider(contextSharingGlObjectsProvider) .setGlObjectsProvider(contextSharingGlObjectsProvider)
.build(); .build();
videoFrameProcessorTestRunner = videoFrameProcessorTestRunner =
...@@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader textureBitmapReader = new TextureBitmapReader(); TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(textureBitmapReader::readBitmapFromTexture) .setTextureOutput(
textureBitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.build(); .build();
return new VideoFrameProcessorTestRunner.Builder() return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId) .setTestId(testId)
...@@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private static final class TextureBitmapReader implements BitmapReader { private static final class TextureBitmapReader implements BitmapReader {
// TODO(b/239172735): This outputs an incorrect black output image on emulators. // TODO(b/239172735): This outputs an incorrect black output image on emulators.
private boolean useHighPrecisionColorComponents; private boolean useHighPrecisionColorComponents;
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
private @MonotonicNonNull Bitmap outputBitmap; private @MonotonicNonNull Bitmap outputBitmap;
@Override
@Nullable @Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { @Override
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener releaseOutputFrameListener) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
this.releaseOutputFrameListener = releaseOutputFrameListener;
return null; return null;
} }
...@@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
} }
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs) public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException { throws VideoFrameProcessingException {
GlUtil.focusFramebufferUsingCurrentContext( try {
outputTexture.fboId, outputTexture.width, outputTexture.height); GlUtil.focusFramebufferUsingCurrentContext(
outputBitmap = outputTexture.fboId, outputTexture.width, outputTexture.height);
createBitmapFromCurrentGlFrameBuffer( outputBitmap =
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents); createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
} }
private static Bitmap createBitmapFromCurrentGlFrameBuffer( private static Bitmap createBitmapFromCurrentGlFrameBuffer(
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java
...@@ -283,13 +283,14 @@ public final class VideoFrameProcessorTestRunner { ...@@ -283,13 +283,14 @@ public final class VideoFrameProcessorTestRunner {
new VideoFrameProcessor.Listener() { new VideoFrameProcessor.Listener() {
@Override @Override
public void onOutputSizeChanged(int width, int height) { public void onOutputSizeChanged(int width, int height) {
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
@Nullable @Nullable
Surface outputSurface = Surface outputSurface =
bitmapReader.getSurface( bitmapReader.getSurface(
width, width,
height, height,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr( useHighPrecisionColorComponents,
outputColorInfo)); checkNotNull(videoFrameProcessor)::releaseOutputFrame);
if (outputSurface != null) { if (outputSurface != null) {
checkNotNull(videoFrameProcessor) checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height)); .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
...@@ -404,10 +405,18 @@ public final class VideoFrameProcessorTestRunner { ...@@ -404,10 +405,18 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */ /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader { public interface BitmapReader {
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
interface ReleaseOutputFrameListener {
void releaseOutputFrame(long releaseTimeUs);
}
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */ /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable @Nullable
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents); Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener);
/** Returns the output {@link Bitmap}. */ /** Returns the output {@link Bitmap}. */
Bitmap getBitmap(); Bitmap getBitmap();
...@@ -427,7 +436,11 @@ public final class VideoFrameProcessorTestRunner { ...@@ -427,7 +436,11 @@ public final class VideoFrameProcessorTestRunner {
@Override @Override
@SuppressLint("WrongConstant") @SuppressLint("WrongConstant")
@Nullable @Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener) {
imageReader = imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
return imageReader.getSurface(); return imageReader.getSurface();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment