Skip to content

SDK / exoplayer

Go to a project
Commit 719715d2 by huangdarwin Committed by Tofunmi Adigun-Hameed
Options

Effect: Multiple Texture output 

Allow the VideoFrameProcessor to output multiple textures at a time, so that
lifetime of textures is up to the consumer calling VFP.releaseOutputFrame.

The FinalShaderProgramWrapper also has a new maxCapacity limit added, to ensure
the a reasonable amount of textures is used and avoid using up memory.

PiperOrigin-RevId: 532094256
(cherry picked from commit 025f71bec45e06b19b7237967af2677b84387b08)
parent 568233f6
Expand all
Inline Side-by-side
Showing with 188 additions and 62 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
......@@ -143,7 +143,7 @@ public interface VideoFrameProcessor {
*/
void onError(VideoFrameProcessingException exception);
/** Called after the {@link VideoFrameProcessor} has produced its final output frame. */
/** Called after the {@link VideoFrameProcessor} has rendered its final output frame. */
void onEnded();
}
......@@ -290,6 +290,18 @@ public interface VideoFrameProcessor {
void renderOutputFrame(long renderTimeNs);
/**
* Releases resources associated with all output frames with presentation time less than or equal
* to {@code presentationTimeUs}.
*
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
* be required for other outputs.
*
* @param presentationTimeUs The presentation time where all frames before and at this time should
* be released, in microseconds.
*/
void releaseOutputFrame(long presentationTimeUs);
/**
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
*
* <p>Can be called on any thread.
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
......@@ -31,6 +31,7 @@ import android.opengl.GLES20;
import android.opengl.GLES30;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
......@@ -71,7 +72,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public interface TextureOutputListener {
/** Called when a texture has been rendered to. */
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException;
throws VideoFrameProcessingException;
}
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
......@@ -82,6 +83,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private boolean enableColorTransfers;
private GlObjectsProvider glObjectsProvider;
@Nullable private TextureOutputListener textureOutputListener;
private int textureOutputCapacity;
/** Creates an instance. */
public Builder() {
......@@ -112,39 +114,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
}
/**
* Sets the {@link TextureOutputListener}.
* Sets texture output settings.
*
* <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via
* {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to.
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
* {@code textureOutputCapacity} is reached, until they're released via {@link
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
*
* <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will
* be be adjusted as needed, to match the output surface's output.
* <p>If not set, there will be no texture output.
*
* <p>This must not be set if the {@linkplain #setOutputSurfaceInfo output surface info} is
* also set.
*
* @param textureOutputListener The {@link TextureOutputListener}.
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
* before texture output blocks. Must be greater than or equal to 1.
*/
@VisibleForTesting
@CanIgnoreReturnValue
public Builder setOnTextureRenderedListener(TextureOutputListener textureOutputListener) {
public Builder setTextureOutput(
TextureOutputListener textureOutputListener,
@IntRange(from = 1) int textureOutputCapacity) {
// TODO: http://b/262694346 - Add tests for multiple texture output.
this.textureOutputListener = textureOutputListener;
checkArgument(textureOutputCapacity >= 1);
this.textureOutputCapacity = textureOutputCapacity;
return this;
}
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
public DefaultVideoFrameProcessor.Factory build() {
return new DefaultVideoFrameProcessor.Factory(
enableColorTransfers, glObjectsProvider, textureOutputListener);
enableColorTransfers, glObjectsProvider, textureOutputListener, textureOutputCapacity);
}
}
private final boolean enableColorTransfers;
private final GlObjectsProvider glObjectsProvider;
@Nullable private final TextureOutputListener textureOutputListener;
private final int textureOutputCapacity;
private Factory(
boolean enableColorTransfers,
GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener) {
@Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity) {
this.enableColorTransfers = enableColorTransfers;
this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener;
this.textureOutputCapacity = textureOutputCapacity;
}
/**
......@@ -229,7 +247,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
listenerExecutor,
listener,
glObjectsProvider,
textureOutputListener));
textureOutputListener,
textureOutputCapacity));
try {
return defaultVideoFrameProcessorFuture.get();
......@@ -409,11 +428,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return checkNotNull(textureManager).getPendingFrameCount();
}
/**
* {@inheritDoc}
*
* <p>This must not be set on an instance where {@linkplain Factory.Builder#setTextureOutput
* texture output} is set.
*/
@Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
}
/**
* {@inheritDoc}
*
* <p>This may also be used for rendering from an output texture, if a {@link
* TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set}
*/
@Override
public void renderOutputFrame(long renderTimeNs) {
checkState(
......@@ -423,6 +454,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
}
/**
* {@inheritDoc}
*
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
* this must be called to release the output information stored in the {@link GlTextureInfo}
* instances.
*/
@Override
public void releaseOutputFrame(long presentationTimeUs) {
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
// input is possible.
videoFrameProcessingTaskExecutor.submit(
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
}
@Override
public void signalEndOfInput() {
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
......@@ -509,7 +555,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor,
Listener listener,
GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener)
@Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws GlUtil.GlException, VideoFrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
......@@ -566,7 +613,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor,
listener,
glObjectsProvider,
textureOutputListener);
textureOutputListener,
textureOutputCapacity);
inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
......@@ -616,7 +664,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor executor,
Listener listener,
GlObjectsProvider glObjectsProvider,
@Nullable TextureOutputListener textureOutputListener)
@Nullable TextureOutputListener textureOutputListener,
int textureOutputCapacity)
throws VideoFrameProcessingException {
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
......@@ -668,7 +717,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
executor,
listener,
glObjectsProvider,
textureOutputListener));
textureOutputListener,
textureOutputCapacity));
return shaderProgramListBuilder.build();
}
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
......@@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs))
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
......@@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs))
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
......@@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(bitmapReader::readBitmapFromTexture)
.setTextureOutput(bitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.setGlObjectsProvider(contextSharingGlObjectsProvider)
.build();
videoFrameProcessorTestRunner =
......@@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setOnTextureRenderedListener(textureBitmapReader::readBitmapFromTexture)
.setTextureOutput(
textureBitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
.build();
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
......@@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private static final class TextureBitmapReader implements BitmapReader {
// TODO(b/239172735): This outputs an incorrect black output image on emulators.
private boolean useHighPrecisionColorComponents;
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
private @MonotonicNonNull Bitmap outputBitmap;
@Override
@Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
@Override
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener releaseOutputFrameListener) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
this.releaseOutputFrameListener = releaseOutputFrameListener;
return null;
}
......@@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
throws GlUtil.GlException {
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
outputBitmap =
createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
throws VideoFrameProcessingException {
try {
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
outputBitmap =
createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
}
private static Bitmap createBitmapFromCurrentGlFrameBuffer(
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java
......@@ -283,13 +283,14 @@ public final class VideoFrameProcessorTestRunner {
new VideoFrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
@Nullable
Surface outputSurface =
bitmapReader.getSurface(
width,
height,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(
outputColorInfo));
useHighPrecisionColorComponents,
checkNotNull(videoFrameProcessor)::releaseOutputFrame);
if (outputSurface != null) {
checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
......@@ -404,10 +405,18 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader {
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
interface ReleaseOutputFrameListener {
void releaseOutputFrame(long releaseTimeUs);
}
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener);
/** Returns the output {@link Bitmap}. */
Bitmap getBitmap();
......@@ -427,7 +436,11 @@ public final class VideoFrameProcessorTestRunner {
@Override
@SuppressLint("WrongConstant")
@Nullable
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
public Surface getSurface(
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener) {
imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
return imageReader.getSurface();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment