Skip to content

SDK / exoplayer

Go to a project
Commit 884e3763 by Googler Committed by microkatz
Options

Allow frame release to be controlled outside FrameProcessor. 

Adds a method to FrameProcessor.Listener to be called when an
output frame is available and a method releaseOutputFrame in
FrameProcessor allowing the caller to trigger release of the
oldest available output frame at a given timestamp. Late frames
or frames with unset release times are dropped in the
FinalMatrixTransformationProcessorWrapper.

More than one output frame can become available before they are
released if the penultimate GlTextureProcessor is capable of producing
multiple output frames. Processing continues while waiting for
releaseOutputFrame to be called. Frame release tasks are prioritized
over other tasks.

PiperOrigin-RevId: 468473072
(cherry picked from commit 2c063546)
parent 4c36dae9
Expand all
Inline Side-by-side
Showing with 75 additions and 9 deletions
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
...@@ -477,6 +477,11 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -477,6 +477,11 @@ public final class GlEffectsFrameProcessorPixelTest {
} }
@Override @Override
public void onOutputFrameAvailable(long presentationTimeNs) {
// Do nothing as frames are released automatically.
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) { public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception); frameProcessingException.set(exception);
} }
...@@ -488,7 +493,8 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -488,7 +493,8 @@ public final class GlEffectsFrameProcessorPixelTest {
}, },
effects, effects,
DebugViewProvider.NONE, DebugViewProvider.NONE,
ColorInfo.SDR_BT709_LIMITED)); ColorInfo.SDR_BT709_LIMITED,
/* releaseFramesAutomatically= */ true));
glEffectsFrameProcessor.setInputFrameInfo( glEffectsFrameProcessor.setInputFrameInfo(
new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0)); new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0));
glEffectsFrameProcessor.registerInputFrame(); glEffectsFrameProcessor.registerInputFrame();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
...@@ -65,7 +65,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -65,7 +65,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo colorInfo) ColorInfo colorInfo,
boolean releaseFramesAutomatically)
throws FrameProcessingException { throws FrameProcessingException {
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
...@@ -79,6 +80,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -79,6 +80,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
effects, effects,
debugViewProvider, debugViewProvider,
colorInfo, colorInfo,
releaseFramesAutomatically,
singleThreadExecutorService)); singleThreadExecutorService));
try { try {
...@@ -109,6 +111,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -109,6 +111,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo colorInfo, ColorInfo colorInfo,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService) ExecutorService singleThreadExecutorService)
throws GlUtil.GlException, FrameProcessingException { throws GlUtil.GlException, FrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
...@@ -133,13 +136,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -133,13 +136,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList<GlTextureProcessor> textureProcessors = ImmutableList<GlTextureProcessor> textureProcessors =
getGlTextureProcessorsForGlEffects( getGlTextureProcessorsForGlEffects(
context, effects, eglDisplay, eglContext, listener, debugViewProvider, colorInfo); context,
effects,
eglDisplay,
eglContext,
listener,
debugViewProvider,
colorInfo,
releaseFramesAutomatically);
FrameProcessingTaskExecutor frameProcessingTaskExecutor = FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener); chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
return new GlEffectsFrameProcessor( return new GlEffectsFrameProcessor(
eglDisplay, eglContext, frameProcessingTaskExecutor, textureProcessors); eglDisplay,
eglContext,
frameProcessingTaskExecutor,
textureProcessors,
releaseFramesAutomatically);
} }
/** /**
...@@ -161,7 +175,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -161,7 +175,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
EGLContext eglContext, EGLContext eglContext,
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo colorInfo) ColorInfo colorInfo,
boolean releaseFramesAutomatically)
throws FrameProcessingException { throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder = ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
...@@ -242,7 +257,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -242,7 +257,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
listener, listener,
debugViewProvider, debugViewProvider,
sampleFromExternalTexture, sampleFromExternalTexture,
colorInfo)); colorInfo,
releaseFramesAutomatically));
return textureProcessorListBuilder.build(); return textureProcessorListBuilder.build();
} }
...@@ -276,6 +292,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -276,6 +292,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final ExternalTextureManager inputExternalTextureManager; private final ExternalTextureManager inputExternalTextureManager;
private final Surface inputSurface; private final Surface inputSurface;
private final boolean releaseFramesAutomatically;
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
private final ImmutableList<GlTextureProcessor> allTextureProcessors; private final ImmutableList<GlTextureProcessor> allTextureProcessors;
...@@ -291,12 +308,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -291,12 +308,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
EGLContext eglContext, EGLContext eglContext,
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor,
ImmutableList<GlTextureProcessor> textureProcessors) ImmutableList<GlTextureProcessor> textureProcessors,
boolean releaseFramesAutomatically)
throws FrameProcessingException { throws FrameProcessingException {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
this.releaseFramesAutomatically = releaseFramesAutomatically;
checkState(!textureProcessors.isEmpty()); checkState(!textureProcessors.isEmpty());
checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor); checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor);
...@@ -348,6 +367,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -348,6 +367,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
} }
@Override @Override
public void releaseOutputFrame(long releaseTimeNs) {
checkState(
!releaseFramesAutomatically,
"Calling this method is not allowed when releaseFramesAutomatically is enabled");
frameProcessingTaskExecutor.submitWithHighPriority(
() -> finalTextureProcessorWrapper.releaseOutputFrame(releaseTimeNs));
}
@Override
public void signalEndOfInput() { public void signalEndOfInput() {
checkState(!inputStreamEnded); checkState(!inputStreamEnded);
inputStreamEnded = true; inputStreamEnded = true;
......
library/common/src/main/java/androidx/media3/common/FrameProcessor.java
...@@ -45,6 +45,10 @@ public interface FrameProcessor { ...@@ -45,6 +45,10 @@ public interface FrameProcessor {
* @param effects The {@link Effect} instances to apply to each frame. * @param effects The {@link Effect} instances to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}. * @param debugViewProvider A {@link DebugViewProvider}.
* @param colorInfo The {@link ColorInfo} for input and output frames. * @param colorInfo The {@link ColorInfo} for input and output frames.
* @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will release
* output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface}
* automatically as they become available. If {@code false}, the {@link FrameProcessor} will
* wait to release each frame until {@link #releaseOutputFrame(long)} is called.
* @return A new instance. * @return A new instance.
* @throws FrameProcessingException If a problem occurs while creating the {@link * @throws FrameProcessingException If a problem occurs while creating the {@link
* FrameProcessor}. * FrameProcessor}.
...@@ -54,7 +58,8 @@ public interface FrameProcessor { ...@@ -54,7 +58,8 @@ public interface FrameProcessor {
Listener listener, Listener listener,
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
ColorInfo colorInfo) ColorInfo colorInfo,
boolean releaseFramesAutomatically)
throws FrameProcessingException; throws FrameProcessingException;
} }
...@@ -74,6 +79,13 @@ public interface FrameProcessor { ...@@ -74,6 +79,13 @@ public interface FrameProcessor {
void onOutputSizeChanged(int width, int height); void onOutputSizeChanged(int width, int height);
/** /**
* Called when an output frame with the given {@code presentationTimeNs} becomes available.
*
* @param presentationTimeNs The presentation time of the frame, in nanoseconds.
*/
void onOutputFrameAvailable(long presentationTimeNs);
/**
* Called when an exception occurs during asynchronous frame processing. * Called when an exception occurs during asynchronous frame processing.
* *
* <p>If an error occurred, consuming and producing further frames will not work as expected and * <p>If an error occurred, consuming and producing further frames will not work as expected and
...@@ -136,6 +148,20 @@ public interface FrameProcessor { ...@@ -136,6 +148,20 @@ public interface FrameProcessor {
void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo); void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo);
/** /**
* Releases the oldest unreleased output frame that has become {@linkplain
* Listener#onOutputFrameAvailable(long) available} at the given {@code releaseTimeNs}.
*
* <p>This method must only be called if {@code releaseFramesAutomatically} was set to {@code
* false} using the {@link Factory} and should be called exactly once for each frame that becomes
* {@linkplain Listener#onOutputFrameAvailable(long) available}.
*
* @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link
* C#TIME_UNSET} to drop the frame. If {@code releaseTimeNs} is after {@link
* System#nanoTime()} at the time of the release, the frame is also dropped.
*/
void releaseOutputFrame(long releaseTimeNs);
/**
* Informs the {@code FrameProcessor} that no further input frames should be accepted. * Informs the {@code FrameProcessor} that no further input frames should be accepted.
* *
* @throws IllegalStateException If called more than once. * @throws IllegalStateException If called more than once.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -126,6 +126,11 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -126,6 +126,11 @@ import org.checkerframework.dataflow.qual.Pure;
} }
@Override @Override
public void onOutputFrameAvailable(long presentationTimeNs) {
// Do nothing as frames are released automatically.
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) { public void onFrameProcessingError(FrameProcessingException exception) {
asyncErrorListener.onTransformationException( asyncErrorListener.onTransformationException(
TransformationException.createForFrameProcessingException( TransformationException.createForFrameProcessingException(
...@@ -147,7 +152,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -147,7 +152,8 @@ import org.checkerframework.dataflow.qual.Pure;
// This implies that the OpenGL EXT_YUV_target extension is supported and hence the // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
// mapping is applied, which ensures the decoder outputs SDR output for an HDR input. // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
encoderWrapper.getSupportedInputColor()); encoderWrapper.getSupportedInputColor(),
/* releaseFramesAutomatically= */ true);
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException( throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED); e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment