Commit 4c2d5467 by huangdarwin Committed by Tofunmi Adigun-Hameed

Effect: Use callback to release texture

This allows us to avoid needing a reference to the VideoFrameProcessor, which
can be especially difficult if an App only has a reference to the
VideoFrameProcessor.Factory it passes into Transformer/ExoPlayer.

PiperOrigin-RevId: 533205983
(cherry picked from commit 4d7c57ed6ee72c633b4e3eb7fff3aa7274b127ce)
parent 40c5b58c
...@@ -47,7 +47,7 @@ public interface VideoFrameProcessor { ...@@ -47,7 +47,7 @@ public interface VideoFrameProcessor {
// TODO(b/243036513): Allow effects to be replaced. // TODO(b/243036513): Allow effects to be replaced.
/** A listener for frame processing events. */ /** A listener for frame processing events. */
public interface OnInputFrameProcessedListener { interface OnInputFrameProcessedListener {
/** Called when the given input frame has been processed. */ /** Called when the given input frame has been processed. */
void onInputFrameProcessed(int textureId) throws VideoFrameProcessingException; void onInputFrameProcessed(int textureId) throws VideoFrameProcessingException;
...@@ -290,18 +290,6 @@ public interface VideoFrameProcessor { ...@@ -290,18 +290,6 @@ public interface VideoFrameProcessor {
void renderOutputFrame(long renderTimeNs); void renderOutputFrame(long renderTimeNs);
/** /**
* Releases resources associated with all output frames with presentation time less than or equal
* to {@code presentationTimeUs}.
*
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
* be required for other outputs.
*
* @param presentationTimeUs The presentation time where all frames before and at this time should
* be released, in microseconds.
*/
void releaseOutputFrame(long presentationTimeUs);
/**
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted. * Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
* *
* <p>Can be called on any thread. * <p>Can be called on any thread.
......
...@@ -69,11 +69,24 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -69,11 +69,24 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** Listener interface for texture output. */ /** Listener interface for texture output. */
@VisibleForTesting(otherwise = PACKAGE_PRIVATE) @VisibleForTesting(otherwise = PACKAGE_PRIVATE)
public interface TextureOutputListener { public interface TextureOutputListener {
/** Called when a texture has been rendered to. */ /**
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs) * Called when a texture has been rendered to. {@code releaseOutputTextureCallback} must be
* called to release the {@link GlTextureInfo}.
*/
void onTextureRendered(
GlTextureInfo outputTexture,
long presentationTimeUs,
ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException; throws VideoFrameProcessingException;
} }
/**
* Releases the output information stored for textures before and at {@code presentationTimeUs}.
*/
public interface ReleaseOutputTextureCallback {
void release(long presentationTimeUs);
}
/** A factory for {@link DefaultVideoFrameProcessor} instances. */ /** A factory for {@link DefaultVideoFrameProcessor} instances. */
public static final class Factory implements VideoFrameProcessor.Factory { public static final class Factory implements VideoFrameProcessor.Factory {
...@@ -116,9 +129,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -116,9 +129,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* Sets texture output settings. * Sets texture output settings.
* *
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via * <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when * {@link TextureOutputListener#onTextureRendered}. Textures will stop being outputted when
* {@code textureOutputCapacity} is reached, until they're released via {@link * the number of output textures available reaches the {@code textureOutputCapacity}. To
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}. * regain capacity, output textures must be released using {@link
* ReleaseOutputTextureCallback}.
* *
* <p>If not set, there will be no texture output. * <p>If not set, there will be no texture output.
* *
...@@ -453,21 +467,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -453,21 +467,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs)); () -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
} }
/**
* {@inheritDoc}
*
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
* this must be called to release the output information stored in the {@link GlTextureInfo}
* instances.
*/
@Override
public void releaseOutputFrame(long presentationTimeUs) {
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
// input is possible.
videoFrameProcessingTaskExecutor.submit(
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
}
@Override @Override
public void signalEndOfInput() { public void signalEndOfInput() {
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos(); DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
...@@ -608,6 +607,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -608,6 +607,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo, outputColorInfo,
enableColorTransfers, enableColorTransfers,
renderFramesAutomatically, renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
...@@ -659,6 +659,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -659,6 +659,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean enableColorTransfers, boolean enableColorTransfers,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor executor, Executor executor,
Listener listener, Listener listener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
...@@ -712,6 +713,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -712,6 +713,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
outputColorInfo, outputColorInfo,
enableColorTransfers, enableColorTransfers,
renderFramesAutomatically, renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
executor, executor,
listener, listener,
glObjectsProvider, glObjectsProvider,
......
...@@ -85,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -85,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ColorInfo outputColorInfo; private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers; private final boolean enableColorTransfers;
private final boolean renderFramesAutomatically; private final boolean renderFramesAutomatically;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final Executor videoFrameProcessorListenerExecutor; private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener; private final VideoFrameProcessor.Listener videoFrameProcessorListener;
private final Queue<Pair<GlTextureInfo, Long>> availableFrames; private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
...@@ -126,6 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -126,6 +127,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean enableColorTransfers, boolean enableColorTransfers,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor, Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener, VideoFrameProcessor.Listener videoFrameProcessorListener,
GlObjectsProvider glObjectsProvider, GlObjectsProvider glObjectsProvider,
...@@ -140,6 +142,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -140,6 +142,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.outputColorInfo = outputColorInfo; this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers; this.enableColorTransfers = enableColorTransfers;
this.renderFramesAutomatically = renderFramesAutomatically; this.renderFramesAutomatically = renderFramesAutomatically;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor; this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
this.videoFrameProcessorListener = videoFrameProcessorListener; this.videoFrameProcessorListener = videoFrameProcessorListener;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
...@@ -224,6 +227,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -224,6 +227,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
public void releaseOutputFrame(long presentationTimeUs) { public void releaseOutputFrame(long presentationTimeUs) {
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
}
private void releaseOutputFrameInternal(long presentationTimeUs) {
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
&& checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) {
outputTexturePool.freeTexture(); outputTexturePool.freeTexture();
...@@ -232,16 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -232,16 +239,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
public void renderOutputFrame(long renderTimeNs) {
frameProcessingStarted = true;
checkState(!renderFramesAutomatically);
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrame(
/* inputTexture= */ oldestAvailableFrame.first,
/* presentationTimeUs= */ oldestAvailableFrame.second,
renderTimeNs);
}
@Override @Override
public void flush() { public void flush() {
frameProcessingStarted = true; frameProcessingStarted = true;
...@@ -267,6 +264,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -267,6 +264,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
public void renderOutputFrame(long renderTimeNs) {
frameProcessingStarted = true;
checkState(!renderFramesAutomatically);
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrame(
/* inputTexture= */ oldestAvailableFrame.first,
/* presentationTimeUs= */ oldestAvailableFrame.second,
renderTimeNs);
}
/** /**
* Sets the output {@link SurfaceInfo}. * Sets the output {@link SurfaceInfo}.
* *
...@@ -369,7 +376,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -369,7 +376,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// glFinish. Consider removing glFinish and requiring onTextureRendered to handle // glFinish. Consider removing glFinish and requiring onTextureRendered to handle
// synchronization. // synchronization.
GLES20.glFinish(); GLES20.glFinish();
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs); checkNotNull(textureOutputListener)
.onTextureRendered(outputTexture, presentationTimeUs, this::releaseOutputFrame);
} }
/** /**
......
...@@ -109,6 +109,8 @@ import java.util.Queue; ...@@ -109,6 +109,8 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use. * <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
*/ */
public void freeTexture(GlTextureInfo textureInfo) { public void freeTexture(GlTextureInfo textureInfo) {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(inUseTextures.contains(textureInfo)); checkState(inUseTextures.contains(textureInfo));
inUseTextures.remove(textureInfo); inUseTextures.remove(textureInfo);
freeTextures.add(textureInfo); freeTextures.add(textureInfo);
...@@ -120,6 +122,8 @@ import java.util.Queue; ...@@ -120,6 +122,8 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if there's no textures in use to free. * <p>Throws {@link IllegalStateException} if there's no textures in use to free.
*/ */
public void freeTexture() { public void freeTexture() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(!inUseTextures.isEmpty()); checkState(!inUseTextures.isEmpty());
GlTextureInfo texture = inUseTextures.remove(); GlTextureInfo texture = inUseTextures.remove();
freeTextures.add(texture); freeTextures.add(texture);
...@@ -127,6 +131,8 @@ import java.util.Queue; ...@@ -127,6 +131,8 @@ import java.util.Queue;
/** Free all in-use textures. */ /** Free all in-use textures. */
public void freeAllTextures() { public void freeAllTextures() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
freeTextures.addAll(inUseTextures); freeTextures.addAll(inUseTextures);
inUseTextures.clear(); inUseTextures.clear();
} }
......
...@@ -507,14 +507,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -507,14 +507,15 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder() new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput( .setTextureOutput(
(outputTexture, presentationTimeUs) -> (outputTexture, presentationTimeUs, releaseOutputTextureCallback) ->
inputTextureIntoVideoFrameProcessor( inputTextureIntoVideoFrameProcessor(
testId, testId,
consumersBitmapReader, consumersBitmapReader,
colorInfo, colorInfo,
effects, effects,
outputTexture, outputTexture,
presentationTimeUs), presentationTimeUs,
releaseOutputTextureCallback),
/* textureOutputCapacity= */ 1) /* textureOutputCapacity= */ 1)
.build(); .build();
return new VideoFrameProcessorTestRunner.Builder() return new VideoFrameProcessorTestRunner.Builder()
...@@ -533,7 +534,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -533,7 +534,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
ColorInfo colorInfo, ColorInfo colorInfo,
List<Effect> effects, List<Effect> effects,
GlTextureInfo texture, GlTextureInfo texture,
long presentationTimeUs) long presentationTimeUs,
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
GlObjectsProvider contextSharingGlObjectsProvider = GlObjectsProvider contextSharingGlObjectsProvider =
new DefaultGlObjectsProvider(GlUtil.getCurrentContext()); new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
...@@ -559,6 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -559,6 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
} catch (InterruptedException e) { } catch (InterruptedException e) {
throw new VideoFrameProcessingException(e); throw new VideoFrameProcessingException(e);
} }
releaseOutputTextureCallback.release(presentationTimeUs);
} }
private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder( private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
...@@ -584,19 +587,12 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -584,19 +587,12 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
private static final class TextureBitmapReader implements BitmapReader { private static final class TextureBitmapReader implements BitmapReader {
// TODO(b/239172735): This outputs an incorrect black output image on emulators. // TODO(b/239172735): This outputs an incorrect black output image on emulators.
private boolean useHighPrecisionColorComponents; private boolean useHighPrecisionColorComponents;
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
private @MonotonicNonNull Bitmap outputBitmap; private @MonotonicNonNull Bitmap outputBitmap;
@Nullable @Nullable
@Override @Override
public Surface getSurface( public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener releaseOutputFrameListener) {
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
this.releaseOutputFrameListener = releaseOutputFrameListener;
return null; return null;
} }
...@@ -605,7 +601,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -605,7 +601,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
return checkStateNotNull(outputBitmap); return checkStateNotNull(outputBitmap);
} }
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs) public void readBitmapFromTexture(
GlTextureInfo outputTexture,
long presentationTimeUs,
DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
try { try {
GlUtil.focusFramebufferUsingCurrentContext( GlUtil.focusFramebufferUsingCurrentContext(
...@@ -613,12 +612,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { ...@@ -613,12 +612,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
outputBitmap = outputBitmap =
createBitmapFromCurrentGlFrameBuffer( createBitmapFromCurrentGlFrameBuffer(
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents); outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) { } catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e); throw new VideoFrameProcessingException(e);
} }
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs); releaseOutputTextureCallback.release(presentationTimeUs);
} }
private static Bitmap createBitmapFromCurrentGlFrameBuffer( private static Bitmap createBitmapFromCurrentGlFrameBuffer(
......
...@@ -286,11 +286,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -286,11 +286,7 @@ public final class VideoFrameProcessorTestRunner {
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo); boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
@Nullable @Nullable
Surface outputSurface = Surface outputSurface =
bitmapReader.getSurface( bitmapReader.getSurface(width, height, useHighPrecisionColorComponents);
width,
height,
useHighPrecisionColorComponents,
checkNotNull(videoFrameProcessor)::releaseOutputFrame);
if (outputSurface != null) { if (outputSurface != null) {
checkNotNull(videoFrameProcessor) checkNotNull(videoFrameProcessor)
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height)); .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
...@@ -405,18 +401,10 @@ public final class VideoFrameProcessorTestRunner { ...@@ -405,18 +401,10 @@ public final class VideoFrameProcessorTestRunner {
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */ /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
public interface BitmapReader { public interface BitmapReader {
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
interface ReleaseOutputFrameListener {
void releaseOutputFrame(long releaseTimeUs);
}
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */ /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
@Nullable @Nullable
Surface getSurface( Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener);
/** Returns the output {@link Bitmap}. */ /** Returns the output {@link Bitmap}. */
Bitmap getBitmap(); Bitmap getBitmap();
...@@ -436,11 +424,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -436,11 +424,7 @@ public final class VideoFrameProcessorTestRunner {
@Override @Override
@SuppressLint("WrongConstant") @SuppressLint("WrongConstant")
@Nullable @Nullable
public Surface getSurface( public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
int width,
int height,
boolean useHighPrecisionColorComponents,
ReleaseOutputFrameListener listener) {
imageReader = imageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
return imageReader.getSurface(); return imageReader.getSurface();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment