Allow frame release to be controlled outside FrameProcessor.

Adds a method to FrameProcessor.Listener to be called when an
output frame is available and a method releaseOutputFrame in
FrameProcessor allowing the caller to trigger release of the
oldest available output frame at a given timestamp. Late frames
or frames with unset release times are dropped in the
FinalMatrixTransformationProcessorWrapper.

More than one output frame can become available before they are
released if the penultimate GlTextureProcessor is capable of producing
multiple output frames. Processing continues while waiting for
releaseOutputFrame to be called. Frame release tasks are prioritized
over other tasks.

PiperOrigin-RevId: 468473072
(cherry picked from commit 2c063546)

google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java

@@ -477,6 +477,11 @@ public final class GlEffectsFrameProcessorPixelTest {
                         }
 
                         @Override
+                        public void onOutputFrameAvailable(long presentationTimeNs) {
+                          // Do nothing as frames are released automatically.
+                        }
+
+                        @Override
                         public void onFrameProcessingError(FrameProcessingException exception) {
                           frameProcessingException.set(exception);
                         }
@@ -488,7 +493,8 @@ public final class GlEffectsFrameProcessorPixelTest {
                       },
                       effects,
                       DebugViewProvider.NONE,
-                      ColorInfo.SDR_BT709_LIMITED));
+                      ColorInfo.SDR_BT709_LIMITED,
+                      /* releaseFramesAutomatically= */ true));
       glEffectsFrameProcessor.setInputFrameInfo(
           new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0));
       glEffectsFrameProcessor.registerInputFrame();

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java

@@ -65,7 +65,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
         FrameProcessor.Listener listener,
         List<Effect> effects,
         DebugViewProvider debugViewProvider,
-        ColorInfo colorInfo)
+        ColorInfo colorInfo,
+        boolean releaseFramesAutomatically)
         throws FrameProcessingException {
 
       ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
@@ -79,6 +80,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
                       effects,
                       debugViewProvider,
                       colorInfo,
+                      releaseFramesAutomatically,
                       singleThreadExecutorService));
 
       try {
@@ -109,6 +111,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       List<Effect> effects,
       DebugViewProvider debugViewProvider,
       ColorInfo colorInfo,
+      boolean releaseFramesAutomatically,
       ExecutorService singleThreadExecutorService)
       throws GlUtil.GlException, FrameProcessingException {
     checkState(Thread.currentThread().getName().equals(THREAD_NAME));
@@ -133,13 +136,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
 
     ImmutableList<GlTextureProcessor> textureProcessors =
         getGlTextureProcessorsForGlEffects(
-            context, effects, eglDisplay, eglContext, listener, debugViewProvider, colorInfo);
+            context,
+            effects,
+            eglDisplay,
+            eglContext,
+            listener,
+            debugViewProvider,
+            colorInfo,
+            releaseFramesAutomatically);
     FrameProcessingTaskExecutor frameProcessingTaskExecutor =
         new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
     chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
 
     return new GlEffectsFrameProcessor(
-        eglDisplay, eglContext, frameProcessingTaskExecutor, textureProcessors);
+        eglDisplay,
+        eglContext,
+        frameProcessingTaskExecutor,
+        textureProcessors,
+        releaseFramesAutomatically);
   }
 
   /**
@@ -161,7 +175,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       EGLContext eglContext,
       FrameProcessor.Listener listener,
       DebugViewProvider debugViewProvider,
-      ColorInfo colorInfo)
+      ColorInfo colorInfo,
+      boolean releaseFramesAutomatically)
       throws FrameProcessingException {
     ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
         new ImmutableList.Builder<>();
@@ -242,7 +257,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
             listener,
             debugViewProvider,
             sampleFromExternalTexture,
-            colorInfo));
+            colorInfo,
+            releaseFramesAutomatically));
     return textureProcessorListBuilder.build();
   }
 
@@ -276,6 +292,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
   private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
   private final ExternalTextureManager inputExternalTextureManager;
   private final Surface inputSurface;
+  private final boolean releaseFramesAutomatically;
   private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
   private final ImmutableList<GlTextureProcessor> allTextureProcessors;
 
@@ -291,12 +308,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       EGLDisplay eglDisplay,
       EGLContext eglContext,
       FrameProcessingTaskExecutor frameProcessingTaskExecutor,
-      ImmutableList<GlTextureProcessor> textureProcessors)
+      ImmutableList<GlTextureProcessor> textureProcessors,
+      boolean releaseFramesAutomatically)
       throws FrameProcessingException {
 
     this.eglDisplay = eglDisplay;
     this.eglContext = eglContext;
     this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
+    this.releaseFramesAutomatically = releaseFramesAutomatically;
 
     checkState(!textureProcessors.isEmpty());
     checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor);
@@ -348,6 +367,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
   }
 
   @Override
+  public void releaseOutputFrame(long releaseTimeNs) {
+    checkState(
+        !releaseFramesAutomatically,
+        "Calling this method is not allowed when releaseFramesAutomatically is enabled");
+    frameProcessingTaskExecutor.submitWithHighPriority(
+        () -> finalTextureProcessorWrapper.releaseOutputFrame(releaseTimeNs));
+  }
+
+  @Override
   public void signalEndOfInput() {
     checkState(!inputStreamEnded);
     inputStreamEnded = true;

library/common/src/main/java/androidx/media3/common/FrameProcessor.java

@@ -45,6 +45,10 @@ public interface FrameProcessor {
      * @param effects The {@link Effect} instances to apply to each frame.
      * @param debugViewProvider A {@link DebugViewProvider}.
      * @param colorInfo The {@link ColorInfo} for input and output frames.
+     * @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will release
+     *     output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface}
+     *     automatically as they become available. If {@code false}, the {@link FrameProcessor} will
+     *     wait to release each frame until {@link #releaseOutputFrame(long)} is called.
      * @return A new instance.
      * @throws FrameProcessingException If a problem occurs while creating the {@link
      *     FrameProcessor}.
@@ -54,7 +58,8 @@ public interface FrameProcessor {
         Listener listener,
         List<Effect> effects,
         DebugViewProvider debugViewProvider,
-        ColorInfo colorInfo)
+        ColorInfo colorInfo,
+        boolean releaseFramesAutomatically)
         throws FrameProcessingException;
   }
 
@@ -74,6 +79,13 @@ public interface FrameProcessor {
     void onOutputSizeChanged(int width, int height);
 
     /**
+     * Called when an output frame with the given {@code presentationTimeNs} becomes available.
+     *
+     * @param presentationTimeNs The presentation time of the frame, in nanoseconds.
+     */
+    void onOutputFrameAvailable(long presentationTimeNs);
+
+    /**
      * Called when an exception occurs during asynchronous frame processing.
      *
      * <p>If an error occurred, consuming and producing further frames will not work as expected and
@@ -136,6 +148,20 @@ public interface FrameProcessor {
   void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo);
 
   /**
+   * Releases the oldest unreleased output frame that has become {@linkplain
+   * Listener#onOutputFrameAvailable(long) available} at the given {@code releaseTimeNs}.
+   *
+   * <p>This method must only be called if {@code releaseFramesAutomatically} was set to {@code
+   * false} using the {@link Factory} and should be called exactly once for each frame that becomes
+   * {@linkplain Listener#onOutputFrameAvailable(long) available}.
+   *
+   * @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link
+   *     C#TIME_UNSET} to drop the frame. If {@code releaseTimeNs} is after {@link
+   *     System#nanoTime()} at the time of the release, the frame is also dropped.
+   */
+  void releaseOutputFrame(long releaseTimeNs);
+
+  /**
    * Informs the {@code FrameProcessor} that no further input frames should be accepted.
    *
    * @throws IllegalStateException If called more than once.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -126,6 +126,11 @@ import org.checkerframework.dataflow.qual.Pure;
                 }
 
                 @Override
+                public void onOutputFrameAvailable(long presentationTimeNs) {
+                  // Do nothing as frames are released automatically.
+                }
+
+                @Override
                 public void onFrameProcessingError(FrameProcessingException exception) {
                   asyncErrorListener.onTransformationException(
                       TransformationException.createForFrameProcessingException(
@@ -147,7 +152,8 @@ import org.checkerframework.dataflow.qual.Pure;
               // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
               // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
               // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
-              encoderWrapper.getSupportedInputColor());
+              encoderWrapper.getSupportedInputColor(),
+              /* releaseFramesAutomatically= */ true);
     } catch (FrameProcessingException e) {
       throw TransformationException.createForFrameProcessingException(
           e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);