Skip to content

SDK / exoplayer

Go to a project
Commit 719715d2 by huangdarwin Committed by Tofunmi Adigun-Hameed
Options

Effect: Multiple Texture output 

Allow the VideoFrameProcessor to output multiple textures at a time, so that
lifetime of textures is up to the consumer calling VFP.releaseOutputFrame.

The FinalShaderProgramWrapper also has a new maxCapacity limit added, to ensure
the a reasonable amount of textures is used and avoid using up memory.

PiperOrigin-RevId: 532094256
(cherry picked from commit 025f71bec45e06b19b7237967af2677b84387b08)
parent 568233f6
Inline Side-by-side
Showing with 181 additions and 55 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
...@@ -143,7 +143,7 @@ public interface VideoFrameProcessor { ...@@ -143,7 +143,7 @@ public interface VideoFrameProcessor {
*/ */
void onError(VideoFrameProcessingException exception); void onError(VideoFrameProcessingException exception);
/** Called after the {@link VideoFrameProcessor} has produced its final output frame. */ /** Called after the {@link VideoFrameProcessor} has rendered its final output frame. */
void onEnded(); void onEnded();
} }
...@@ -290,6 +290,18 @@ public interface VideoFrameProcessor { ...@@ -290,6 +290,18 @@ public interface VideoFrameProcessor {
void renderOutputFrame(long renderTimeNs); void renderOutputFrame(long renderTimeNs);
/** /**
* Releases resources associated with all output frames with presentation time less than or equal
* to {@code presentationTimeUs}.
*
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
* be required for other outputs.
*
* @param presentationTimeUs The presentation time where all frames before and at this time should
* be released, in microseconds.
*/
void releaseOutputFrame(long presentationTimeUs);
/**
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted. * Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
* *
* <p>Can be called on any thread. * <p>Can be called on any thread.
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
...@@ -31,6 +31,7 @@ import android.opengl.GLES20; ...@@ -31,6 +31,7 @@ import android.opengl.GLES20;
import android.opengl.GLES30; import android.opengl.GLES30;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.GuardedBy; import androidx.annotation.GuardedBy;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
...@@ -71,7 +72,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -71,7 +72,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public interface TextureOutputListener { public interface TextureOutputListener {
/** Called when a texture has been rendered to. */ /** Called when a texture has been rendered to. */
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs) void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException; throws VideoFrameProcessingException;
} }
/** A factory for {@link DefaultVideoFrameProcessor} instances. */ /** A factory for {@link DefaultVideoFrameProcessor} instances. */
...@@ -82,6 +83,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -82,6 +83,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private boolean enableColorTransfers; private boolean enableColorTransfers;
private GlObjectsProvider glObjectsProvider; private GlObjectsProvider glObjectsProvider;
@Nullable private TextureOutputListener textureOutputListener; @Nullable private TextureOutputListener textureOutputListener;
private int textureOutputCapacity;
/** Creates an instance. */ /** Creates an instance. */
public Builder() { public Builder() {
...@@ -112,39 +114,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -112,39 +114,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
} }
/** /**
* Sets the {@link TextureOutputListener}. * Sets texture output settings.
* *
* <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via * <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
* {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to. * {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
* {@code textureOutputCapacity} is reached, until they're released via {@link
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
* *
* <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will * <p>If not set, there will be no texture output.
* be be adjusted as needed, to match the output surface's output. *
* <p>This must not be set if the {@linkplain #setOutputSurfaceInfo output surface info} is
* also set.
*
* @param textureOutputListener The {@link TextureOutputListener}.
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
* before texture output blocks. Must be greater than or equal to 1.
*/ */
@VisibleForTesting @VisibleForTesting
@CanIgnoreReturnValue @CanIgnoreReturnValue
public Builder setOnTextureRenderedListener(TextureOutputListener textureOutputListener) { public Builder setTextureOutput(
TextureOutputListener textureOutputListener,
@IntRange(from = 1) int textureOutputCapacity) {
// TODO: http://b/262694346 - Add tests for multiple texture output.
this.textureOutputListener = textureOutputListener; this.textureOutputListener = textureOutputListener;
checkArgument(textureOutputCapacity >= 1);
this.textureOutputCapacity = textureOutputCapacity;
return this; return this;
} }
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */ /** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
public DefaultVideoFrameProcessor.Factory build() { public DefaultVideoFrameProcessor.Factory build() {
return new DefaultVideoFrameProcessor.Factory( return new DefaultVideoFrameProcessor.Factory(
enableColorTransfers, glObjectsProvider, textureOutputListener); enableColorTransfers, glObjectsProvider, textureOutputListener, textureOutputCapacity);
} }
} }
private final boolean enableColorTransfers; private final boolean enableColorTransfers;
private final GlObjectsProvider glObjectsProvider; private final GlObjectsProvider glObjectsProvider;
@Nullable private final TextureOutputListener textureOutputListener; @Nullable private final TextureOutputListener textureOutputListener;
private final int textureOutputCapacity;
private Factory( private Factory(
boolean enableColorTransfers, boolean enableColorTransfers,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) { @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity) {
this.enableColorTransfers = enableColorTransfers; this.enableColorTransfers = enableColorTransfers;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener; this.textureOutputListener = textureOutputListener;
this.textureOutputCapacity = textureOutputCapacity;
} }
/** /**
...@@ -229,7 +247,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -229,7 +247,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
listenerExecutor, listenerExecutor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener)); textureOutputListener,
textureOutputCapacity));
try { try {
return defaultVideoFrameProcessorFuture.get(); return defaultVideoFrameProcessorFuture.get();
...@@ -409,11 +428,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -409,11 +428,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return checkNotNull(textureManager).getPendingFrameCount(); return checkNotNull(textureManager).getPendingFrameCount();
} }
/**
* {@inheritDoc}
*
* <p>This must not be set on an instance where {@linkplain Factory.Builder#setTextureOutput
* texture output} is set.
*/
@Override @Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo); finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
} }
/**
* {@inheritDoc}
*
* <p>This may also be used for rendering from an output texture, if a {@link
* TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set}
*/
@Override @Override
public void renderOutputFrame(long renderTimeNs) { public void renderOutputFrame(long renderTimeNs) {
checkState( checkState(
...@@ -423,6 +454,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -423,6 +454,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs)); () -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
} }
/**
* {@inheritDoc}
*
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
* this must be called to release the output information stored in the {@link GlTextureInfo}
* instances.
*/
@Override
public void releaseOutputFrame(long presentationTimeUs) {
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
// input is possible.
videoFrameProcessingTaskExecutor.submit(
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
}
@Override @Override
public void signalEndOfInput() { public void signalEndOfInput() {
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos(); DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
...@@ -509,7 +555,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -509,7 +555,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor, Executor executor,
Listener listener, Listener listener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws GlUtil.GlException, VideoFrameProcessingException { throws GlUtil.GlException, VideoFrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
...@@ -566,7 +613,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -566,7 +613,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener); textureOutputListener,
textureOutputCapacity);
inputSwitcher.registerInput(INPUT_TYPE_SURFACE); inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
if (!ColorInfo.isTransferHdr(inputColorInfo)) { if (!ColorInfo.isTransferHdr(inputColorInfo)) {
...@@ -616,7 +664,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -616,7 +664,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor, Executor executor,
Listener listener, Listener listener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) @Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder = ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
...@@ -668,7 +717,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -668,7 +717,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
textureOutputListener)); textureOutputListener,
textureOutputCapacity));
return shaderProgramListBuilder.build(); return shaderProgramListBuilder.build();
} }
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java
...@@ -50,8 +50,8 @@ import java.util.concurrent.Executor; ...@@ -50,8 +50,8 @@ import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* Wrapper around a {@link DefaultShaderProgram} that renders to the provided output surface or * Wrapper around a {@link DefaultShaderProgram} that renders to either the provided output surface
* texture. * or texture.
* *
* <p>Also renders to a debug surface, if provided. * <p>Also renders to a debug surface, if provided.
* *
...@@ -87,17 +87,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -87,17 +87,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Executor videoFrameProcessorListenerExecutor; private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener; private final VideoFrameProcessor.Listener videoFrameProcessorListener;
private final Queue<Pair<GlTextureInfo, Long>> availableFrames; private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
private final Queue<Pair<GlTextureInfo, Long>> outputTextures;
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener; @Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
private final int textureOutputCapacity;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
private int outputWidth;
private int outputHeight;
@Nullable private DefaultShaderProgram defaultShaderProgram; @Nullable private DefaultShaderProgram defaultShaderProgram;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private GlObjectsProvider glObjectsProvider; private GlObjectsProvider glObjectsProvider;
private InputListener inputListener; private InputListener inputListener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@Nullable private SurfaceView debugSurfaceView; @Nullable private SurfaceView debugSurfaceView;
@Nullable private GlTextureInfo outputTexture;
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener; @Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
private boolean frameProcessingStarted; private boolean frameProcessingStarted;
...@@ -125,7 +128,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -125,7 +128,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Executor videoFrameProcessorListenerExecutor, Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener, VideoFrameProcessor.Listener videoFrameProcessorListener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
@Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener) { @Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
int textureOutputCapacity) {
this.context = context; this.context = context;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
this.rgbMatrices = rgbMatrices; this.rgbMatrices = rgbMatrices;
...@@ -139,9 +143,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -139,9 +143,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.videoFrameProcessorListener = videoFrameProcessorListener; this.videoFrameProcessorListener = videoFrameProcessorListener;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener; this.textureOutputListener = textureOutputListener;
this.textureOutputCapacity = textureOutputCapacity;
inputListener = new InputListener() {}; inputListener = new InputListener() {};
availableFrames = new ConcurrentLinkedQueue<>(); availableFrames = new ConcurrentLinkedQueue<>();
outputTextures = new ConcurrentLinkedQueue<>();
} }
@Override @Override
...@@ -155,7 +161,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -155,7 +161,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void setInputListener(InputListener inputListener) { public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener; this.inputListener = inputListener;
inputListener.onReadyToAcceptInputFrame(); maybeOnReadyToAcceptInputFrame();
} }
@Override @Override
...@@ -193,20 +199,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -193,20 +199,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frameProcessingStarted = true; frameProcessingStarted = true;
videoFrameProcessorListenerExecutor.execute( videoFrameProcessorListenerExecutor.execute(
() -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs)); () -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs));
if (textureOutputListener == null) {
if (renderFramesAutomatically) { if (renderFramesAutomatically) {
renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000); renderFrame(
inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
} else { } else {
availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
} }
inputListener.onReadyToAcceptInputFrame(); } else {
checkState(outputTextures.size() < textureOutputCapacity);
renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
}
maybeOnReadyToAcceptInputFrame();
} }
@Override @Override
public void releaseOutputFrame(GlTextureInfo outputTexture) { public void releaseOutputFrame(GlTextureInfo outputTexture) {
// The final shader program writes to a surface so there is no texture to release. // FinalShaderProgramWrapper cannot release output textures using GlTextureInfo.
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public void releaseOutputFrame(long presentationTimeUs) throws VideoFrameProcessingException {
while (!outputTextures.isEmpty()
&& checkNotNull(outputTextures.peek()).second <= presentationTimeUs) {
GlTextureInfo outputTexture = outputTextures.remove().first;
try {
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException exception) {
throw new VideoFrameProcessingException(exception);
}
maybeOnReadyToAcceptInputFrame();
}
}
public void renderOutputFrame(long renderTimeNs) { public void renderOutputFrame(long renderTimeNs) {
frameProcessingStarted = true; frameProcessingStarted = true;
checkState(!renderFramesAutomatically); checkState(!renderFramesAutomatically);
...@@ -226,7 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -226,7 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
defaultShaderProgram.flush(); defaultShaderProgram.flush();
} }
inputListener.onFlush(); inputListener.onFlush();
inputListener.onReadyToAcceptInputFrame(); maybeOnReadyToAcceptInputFrame();
} }
@Override @Override
...@@ -235,8 +261,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -235,8 +261,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
defaultShaderProgram.release(); defaultShaderProgram.release();
} }
try { try {
if (outputTexture != null) { while (!outputTextures.isEmpty()) {
GlTextureInfo outputTexture = checkNotNull(this.outputTexture); GlTextureInfo outputTexture = outputTextures.remove().first;
GlUtil.deleteTexture(outputTexture.texId); GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId); GlUtil.deleteFbo(outputTexture.fboId);
} }
...@@ -252,6 +278,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -252,6 +278,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @see VideoFrameProcessor#setOutputSurfaceInfo(SurfaceInfo) * @see VideoFrameProcessor#setOutputSurfaceInfo(SurfaceInfo)
*/ */
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
checkState(textureOutputListener == null);
if (Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { if (Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
return; return;
} }
...@@ -276,18 +303,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -276,18 +303,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.outputSurfaceInfo = outputSurfaceInfo; this.outputSurfaceInfo = outputSurfaceInfo;
} }
private void maybeOnReadyToAcceptInputFrame() {
if (textureOutputListener == null || outputTextures.size() < textureOutputCapacity) {
inputListener.onReadyToAcceptInputFrame();
}
}
private synchronized void renderFrame( private synchronized void renderFrame(
GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) { GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) {
try { try {
if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|| !ensureConfigured(inputTexture.width, inputTexture.height)) { || !ensureConfigured(inputTexture.width, inputTexture.height)) {
inputListener.onInputFrameProcessed(inputTexture); inputListener.onInputFrameProcessed(inputTexture);
return; // Drop frames when requested, or there is no output surface. return; // Drop frames when requested, or there is no output surface and output texture.
} }
if (outputSurfaceInfo != null) { if (outputSurfaceInfo != null) {
renderFrameToOutputSurface(inputTexture, presentationTimeUs, renderTimeNs); renderFrameToOutputSurface(inputTexture, presentationTimeUs, renderTimeNs);
} } else if (textureOutputListener != null) {
if (textureOutputListener != null) {
renderFrameToOutputTexture(inputTexture, presentationTimeUs); renderFrameToOutputTexture(inputTexture, presentationTimeUs);
} }
} catch (VideoFrameProcessingException | GlUtil.GlException e) { } catch (VideoFrameProcessingException | GlUtil.GlException e) {
...@@ -331,12 +363,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -331,12 +363,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs) private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException { throws GlUtil.GlException, VideoFrameProcessingException {
GlTextureInfo outputTexture = checkNotNull(this.outputTexture); // TODO(b/262694346): Use a texture pool instead of creating a new texture on every frame.
int outputTexId =
GlUtil.createTexture(
outputWidth,
outputHeight,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
GlTextureInfo outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
GlUtil.focusFramebufferUsingCurrentContext( GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height); outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame(); GlUtil.clearOutputFrame();
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs); checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs);
// TODO(b/262694346): If Compositor's VFPs all use the same context, media3 should be able to
// avoid calling glFinish, and require the onTextureRendered listener to decide whether to
// glFinish. Consider removing glFinish and requiring onTextureRendered to handle
// synchronization.
GLES20.glFinish(); GLES20.glFinish();
outputTextures.add(Pair.create(outputTexture, presentationTimeUs));
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs); checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
} }
...@@ -381,11 +426,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -381,11 +426,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return false; return false;
} }
int outputWidth = outputWidth =
outputSurfaceInfo == null outputSurfaceInfo == null
? outputSizeBeforeSurfaceTransformation.getWidth() ? outputSizeBeforeSurfaceTransformation.getWidth()
: outputSurfaceInfo.width; : outputSurfaceInfo.width;
int outputHeight = outputHeight =
outputSurfaceInfo == null outputSurfaceInfo == null
? outputSizeBeforeSurfaceTransformation.getHeight() ? outputSizeBeforeSurfaceTransformation.getHeight()
: outputSurfaceInfo.height; : outputSurfaceInfo.height;
...@@ -411,16 +456,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -411,16 +456,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
this.debugSurfaceView = debugSurfaceView; this.debugSurfaceView = debugSurfaceView;
if (textureOutputListener != null) {
int outputTexId =
GlUtil.createTexture(
outputWidth,
outputHeight,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
}
if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) { if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
defaultShaderProgram.release(); defaultShaderProgram.release();
defaultShaderProgram = null; defaultShaderProgram = null;
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
...@@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener( .setTextureOutput(
(outputTexture, presentationTimeUs) -> (outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor( inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs)) testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build(); .build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder() new VideoFrameProcessorTestRunner.Builder()
...@@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader(); TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener( .setTextureOutput(
(outputTexture, presentationTimeUs) -> (outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor( inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs)) testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build(); .build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner = VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder() new VideoFrameProcessorTestRunner.Builder()
...@@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
new DefaultGlObjectsProvider(GlUtil.getCurrentContext()); new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(bitmapReader::readBitmapFromTexture) .setTextureOutput(bitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.setGlObjectsProvider(contextSharingGlObjectsProvider) .setGlObjectsProvider(contextSharingGlObjectsProvider)
.build(); .build();
videoFrameProcessorTestRunner = videoFrameProcessorTestRunner =
...@@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader textureBitmapReader = new TextureBitmapReader(); TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(textureBitmapReader::readBitmapFromTexture) .setTextureOutput(
textureBitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.build(); .build();
return new VideoFrameProcessorTestRunner.Builder() return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId) .setTestId(testId)
...@@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private static final class TextureBitmapReader implements BitmapReader { private static final class TextureBitmapReader implements BitmapReader {
// TODO(b/239172735): This outputs an incorrect black output image on emulators. // TODO(b/239172735): This outputs an incorrect black output image on emulators.
private boolean useHighPrecisionColorComponents; private boolean useHighPrecisionColorComponents;
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
private @MonotonicNonNull Bitmap outputBitmap; private @MonotonicNonNull Bitmap outputBitmap;
@Override
@Nullable @Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { @Override
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener releaseOutputFrameListener) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
this.releaseOutputFrameListener = releaseOutputFrameListener;
return null; return null;
} }
...@@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
} }
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs) public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException { throws VideoFrameProcessingException {
try {
GlUtil.focusFramebufferUsingCurrentContext( GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height); outputTexture.fboId, outputTexture.width, outputTexture.height);
outputBitmap = outputBitmap =
createBitmapFromCurrentGlFrameBuffer( createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents); outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
} }
private static Bitmap createBitmapFromCurrentGlFrameBuffer( private static Bitmap createBitmapFromCurrentGlFrameBuffer(
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java
...@@ -283,13 +283,14 @@ public final class VideoFrameProcessorTestRunner { ...@@ -283,13 +283,14 @@ public final class VideoFrameProcessorTestRunner {
new VideoFrameProcessor.Listener() { new VideoFrameProcessor.Listener() {
@Override @Override
public void onOutputSizeChanged(int width, int height) { public void onOutputSizeChanged(int width, int height) {
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
@Nullable @Nullable
Surface outputSurface = Surface outputSurface =
bitmapReader.getSurface( bitmapReader.getSurface(
width, width,
height, height,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr( useHighPrecisionColorComponents,
outputColorInfo)); checkNotNull(videoFrameProcessor)::releaseOutputFrame);
if (outputSurface != null) { if (outputSurface != null) {
checkNotNull(videoFrameProcessor) checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height)); .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
...@@ -404,10 +405,18 @@ public final class VideoFrameProcessorTestRunner { ...@@ -404,10 +405,18 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */ /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader { public interface BitmapReader {
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
interface ReleaseOutputFrameListener {
void releaseOutputFrame(long releaseTimeUs);
}
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */ /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable @Nullable
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents); Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener);
/** Returns the output {@link Bitmap}. */ /** Returns the output {@link Bitmap}. */
Bitmap getBitmap(); Bitmap getBitmap();
...@@ -427,7 +436,11 @@ public final class VideoFrameProcessorTestRunner { ...@@ -427,7 +436,11 @@ public final class VideoFrameProcessorTestRunner {
@Override @Override
@SuppressLint("WrongConstant") @SuppressLint("WrongConstant")
@Nullable @Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener) {
imageReader = imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
return imageReader.getSurface(); return imageReader.getSurface();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment