Make Codec release frames to frame processor.

PiperOrigin-RevId: 495406734

library/common/src/main/java/com/google/android/exoplayer2/C.java

@@ -1103,6 +1103,12 @@ public final class C {
    */
    */
   public static final int COLOR_RANGE_FULL = MediaFormat.COLOR_RANGE_FULL;
   public static final int COLOR_RANGE_FULL = MediaFormat.COLOR_RANGE_FULL;
 
 
+  /**
+   * Represents applying no limit to the number of input frames a {@link MediaCodec} encoder
+   * accepts.
+   */
+  public static final int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE;
+
   /** Video projection types. */
   /** Video projection types. */
   @Documented
   @Documented
   @Retention(RetentionPolicy.SOURCE)
   @Retention(RetentionPolicy.SOURCE)

library/common/src/main/java/com/google/android/exoplayer2/util/Util.java

@@ -16,6 +16,7 @@
 package com.google.android.exoplayer2.util;
 package com.google.android.exoplayer2.util;
 
 
 import static android.content.Context.UI_MODE_SERVICE;
 import static android.content.Context.UI_MODE_SERVICE;
+import static com.google.android.exoplayer2.C.UNLIMITED_PENDING_FRAME_COUNT;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_BACK;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_BACK;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_FORWARD;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_FORWARD;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM;
 import static com.google.android.exoplayer2.Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM;
@@ -50,6 +51,7 @@ import android.graphics.Point;
 import android.hardware.display.DisplayManager;
 import android.hardware.display.DisplayManager;
 import android.media.AudioFormat;
 import android.media.AudioFormat;
 import android.media.AudioManager;
 import android.media.AudioManager;
+import android.media.MediaCodec;
 import android.media.MediaDrm;
 import android.media.MediaDrm;
 import android.net.Uri;
 import android.net.Uri;
 import android.os.Build;
 import android.os.Build;
@@ -2671,6 +2673,46 @@ public final class Util {
   }
   }
 
 
   /**
   /**
+   * Returns the number of maximum pending input frames that are allowed on a {@link MediaCodec}
+   * encoder.
+   */
+  public static int getMaxPendingFramesCountForMediaCodecEncoders(
+      Context context, String codecName, boolean requestedHdrToneMapping) {
+    if (SDK_INT < 29
+        || context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) {
+      // Prior to API 29, decoders may drop frames to keep their output surface from growing out of
+      // bounds. From API 29, if the app targets API 29 or later, the {@link
+      // MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is
+      // full.
+      // Frame dropping is never desired, so a workaround is needed for older API levels.
+      // Allow a maximum of one frame to be pending at a time to prevent frame dropping.
+      // TODO(b/226330223): Investigate increasing this limit.
+      return 1;
+    }
+    if (Ascii.toUpperCase(codecName).startsWith("OMX.")) {
+      // Some OMX decoders don't correctly track their number of output buffers available, and get
+      // stuck if too many frames are rendered without being processed, so limit the number of
+      // pending frames to avoid getting stuck. This value is experimentally determined. See also
+      // b/213455700, b/230097284, b/229978305, and b/245491744.
+      //
+      // OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+.
+      return 5;
+    }
+    if (requestedHdrToneMapping
+        && codecName.equals("c2.qti.hevc.decoder")
+        && MODEL.equals("SM-F936B")) {
+      // This decoder gets stuck if too many frames are rendered without being processed when
+      // tone-mapping HDR10. This value is experimentally determined. See also b/260408846.
+      // TODO(b/260713009): Add API version check after bug is fixed on new API versions.
+      return 12;
+    }
+
+    // Otherwise don't limit the number of frames that can be pending at a time, to maximize
+    // throughput.
+    return UNLIMITED_PENDING_FRAME_COUNT;
+  }
+
+  /**
    * Returns string representation of a {@link C.FormatSupport} flag.
    * Returns string representation of a {@link C.FormatSupport} flag.
    *
    *
    * @param formatSupport A {@link C.FormatSupport} flag.
    * @param formatSupport A {@link C.FormatSupport} flag.

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.DISCA
 import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_NO;
 import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_NO;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 import static java.lang.Math.max;
 import static java.lang.Math.max;
 import static java.lang.Math.min;
 import static java.lang.Math.min;
 
 
@@ -937,6 +938,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     if (Util.SDK_INT >= 23 && tunneling) {
     if (Util.SDK_INT >= 23 && tunneling) {
       tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec()));
       tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec()));
     }
     }
+    frameProcessorManager.onCodecInitialized(name);
   }
   }
 
 
   @Override
   @Override
@@ -1137,13 +1139,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
             && (shouldRenderFirstFrame
             && (shouldRenderFirstFrame
                 || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs)));
                 || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs)));
     if (forceRenderOutputBuffer) {
     if (forceRenderOutputBuffer) {
-      long releaseTimeNs = System.nanoTime();
-      notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
-      if (Util.SDK_INT >= 21) {
-        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
+      // TODO(b/238302341): Handle releasing the force rendered frames in FrameProcessor.
+      boolean notifyFrameMetaDataListener;
+      if (frameProcessorManager.isEnabled()) {
+        notifyFrameMetaDataListener = false;
+        if (!frameProcessorManager.maybeRegisterFrame()) {
+          // TODO(b/238302341): Handle FrameProcessor is unable to accept the force rendered buffer.
+          //  Treat the frame as dropped for now.
+          return true;
+        }
       } else {
       } else {
-        renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
+        notifyFrameMetaDataListener = true;
       }
       }
+      renderOutputBufferNow(
+          codec, format, bufferIndex, presentationTimeUs, notifyFrameMetaDataListener);
       updateVideoFrameProcessingOffsetCounters(earlyUs);
       updateVideoFrameProcessingOffsetCounters(earlyUs);
       return true;
       return true;
     }
     }
@@ -1174,6 +1183,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       return true;
       return true;
     }
     }
 
 
+    if (frameProcessorManager.isEnabled()) {
+      frameProcessorManager.releaseProcessedFrames();
+      if (frameProcessorManager.maybeRegisterFrame()) {
+        renderOutputBufferNow(
+            codec,
+            format,
+            bufferIndex,
+            presentationTimeUs,
+            /* notifyFrameMetadataListener= */ false);
+        return true;
+      }
+      return false;
+    }
+
     if (Util.SDK_INT >= 21) {
     if (Util.SDK_INT >= 21) {
       // Let the underlying framework time the release.
       // Let the underlying framework time the release.
       if (earlyUs < 50000) {
       if (earlyUs < 50000) {
@@ -1381,6 +1404,40 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
   }
 
 
   /**
   /**
+   * Renders the output buffer with the specified index now.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param format The {@link Format} associated with the buffer.
+   * @param index The index of the output buffer to drop.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   * @param notifyFrameMetadataListener Whether to notify the {@link VideoFrameMetadataListener}.
+   */
+  private void renderOutputBufferNow(
+      MediaCodecAdapter codec,
+      Format format,
+      int index,
+      long presentationTimeUs,
+      boolean notifyFrameMetadataListener) {
+    // In previewing mode, use the presentation time as release time so that the SurfaceTexture is
+    // accompanied by the rendered frame's presentation time. Setting a realtime based release time
+    // is only relevant when rendering to a SurfaceView (that is when not using FrameProcessor) for
+    // better frame release. In previewing mode MediaCodec renders to FrameProcessor's input
+    // surface, which is not a SurfaceView.
+    long releaseTimeNs =
+        frameProcessorManager.isEnabled()
+            ? (presentationTimeUs + getOutputStreamOffsetUs()) * 1000
+            : System.nanoTime();
+    if (notifyFrameMetadataListener) {
+      notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
+    }
+    if (Util.SDK_INT >= 21) {
+      renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs);
+    } else {
+      renderOutputBuffer(codec, index, presentationTimeUs);
+    }
+  }
+
+  /**
    * Renders the output buffer with the specified index. This method is only called if the platform
    * Renders the output buffer with the specified index. This method is only called if the platform
    * API version of the device is less than 21.
    * API version of the device is less than 21.
    *
    *
@@ -1686,12 +1743,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     private @MonotonicNonNull Handler handler;
     private @MonotonicNonNull Handler handler;
     @Nullable private FrameProcessor frameProcessor;
     @Nullable private FrameProcessor frameProcessor;
     @Nullable private CopyOnWriteArrayList<Effect> videoEffects;
     @Nullable private CopyOnWriteArrayList<Effect> videoEffects;
+    private int frameProcessorMaxPendingFrameCount;
     private boolean canEnableFrameProcessing;
     private boolean canEnableFrameProcessing;
 
 
     /** Creates a new instance. */
     /** Creates a new instance. */
     public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) {
     public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) {
       this.renderer = renderer;
       this.renderer = renderer;
       processedFrames = new ArrayDeque<>();
       processedFrames = new ArrayDeque<>();
+      frameProcessorMaxPendingFrameCount = C.LENGTH_UNSET;
       canEnableFrameProcessing = true;
       canEnableFrameProcessing = true;
     }
     }
 
 
@@ -1832,6 +1891,50 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     }
     }
 
 
     /**
     /**
+     * Must be called when the codec is initialized.
+     *
+     * <p>Sets the {@code frameProcessorMaxPendingFrameCount} based on the {@code codecName}.
+     */
+    public void onCodecInitialized(String codecName) {
+      frameProcessorMaxPendingFrameCount =
+          Util.getMaxPendingFramesCountForMediaCodecEncoders(
+              renderer.context, codecName, /* requestedHdrToneMapping= */ false);
+    }
+
+    /**
+     * Tries to {@linkplain FrameProcessor#registerInputFrame register an input frame}.
+     *
+     * <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
+     * this method.
+     *
+     * @return Whether {@link MediaCodec} should render the frame to {@link FrameProcessor}.
+     */
+    public boolean maybeRegisterFrame() {
+      checkStateNotNull(frameProcessor);
+      checkState(frameProcessorMaxPendingFrameCount != C.LENGTH_UNSET);
+      if (frameProcessor.getPendingInputFrameCount() < frameProcessorMaxPendingFrameCount) {
+        frameProcessor.registerInputFrame();
+        return true;
+      }
+      return false;
+    }
+
+    /**
+     * Releases the processed frames to the {@linkplain #setOutputSurfaceInfo output surface}.
+     *
+     * <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
+     * this method.
+     */
+    public void releaseProcessedFrames() {
+      while (!processedFrames.isEmpty()) {
+        processedFrames.poll();
+        // TODO(b/238302341): Add frame release logic.
+        checkNotNull(frameProcessor)
+            .releaseOutputFrame(FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY);
+      }
+    }
+
+    /**
      * Releases the resources.
      * Releases the resources.
      *
      *
      * <p>Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before
      * <p>Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java

@@ -31,9 +31,6 @@ import java.nio.ByteBuffer;
  * buffers.
  * buffers.
  */
  */
 public interface Codec {
 public interface Codec {
-  /** Default value for the pending frame count, which represents applying no limit. */
-  int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE;
-
   /** A factory for {@linkplain Codec decoder} instances. */
   /** A factory for {@linkplain Codec decoder} instances. */
   interface DecoderFactory {
   interface DecoderFactory {
 
 
@@ -134,10 +131,10 @@ public interface Codec {
 
 
   /**
   /**
    * Returns the maximum number of frames that may be pending in the output {@code Codec} at a time,
    * Returns the maximum number of frames that may be pending in the output {@code Codec} at a time,
-   * or {@link #UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit.
+   * or {@link C#UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit.
    */
    */
   default int getMaxPendingFrameCount() {
   default int getMaxPendingFrameCount() {
-    return UNLIMITED_PENDING_FRAME_COUNT;
+    return C.UNLIMITED_PENDING_FRAME_COUNT;
   }
   }
 
 
   /**
   /**

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java

@@ -20,7 +20,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
-import static com.google.android.exoplayer2.util.Util.MODEL;
 import static com.google.android.exoplayer2.util.Util.SDK_INT;
 import static com.google.android.exoplayer2.util.Util.SDK_INT;
 
 
 import android.content.Context;
 import android.content.Context;
@@ -40,8 +39,8 @@ import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.MediaFormatUtil;
 import com.google.android.exoplayer2.util.MediaFormatUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.TraceUtil;
+import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.android.exoplayer2.video.ColorInfo;
-import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
@@ -136,7 +135,8 @@ public final class DefaultCodec implements Codec {
     this.mediaCodec = mediaCodec;
     this.mediaCodec = mediaCodec;
     this.inputSurface = inputSurface;
     this.inputSurface = inputSurface;
     maxPendingFrameCount =
     maxPendingFrameCount =
-        getMaxPendingFrameCountInternal(context, mediaCodecName, requestedHdrToneMapping);
+        Util.getMaxPendingFramesCountForMediaCodecEncoders(
+            context, mediaCodecName, requestedHdrToneMapping);
   }
   }
 
 
   @Override
   @Override
@@ -464,42 +464,6 @@ public final class DefaultCodec implements Codec {
     TraceUtil.endSection();
     TraceUtil.endSection();
   }
   }
 
 
-  private static int getMaxPendingFrameCountInternal(
-      Context context, String codecName, boolean requestedHdrToneMapping) {
-    if (SDK_INT < 29
-        || context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) {
-      // Prior to API 29, decoders may drop frames to keep their output surface from growing out of
-      // bounds. From API 29, if the app targets API 29 or later, the {@link
-      // MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is
-      // full.
-      // Frame dropping is never desired, so a workaround is needed for older API levels.
-      // Allow a maximum of one frame to be pending at a time to prevent frame dropping.
-      // TODO(b/226330223): Investigate increasing this limit.
-      return 1;
-    }
-    if (Ascii.toUpperCase(codecName).startsWith("OMX.")) {
-      // Some OMX decoders don't correctly track their number of output buffers available, and get
-      // stuck if too many frames are rendered without being processed, so limit the number of
-      // pending frames to avoid getting stuck. This value is experimentally determined. See also
-      // b/213455700, b/230097284, b/229978305, and b/245491744.
-      //
-      // OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+.
-      return 5;
-    }
-    if (requestedHdrToneMapping
-        && codecName.equals("c2.qti.hevc.decoder")
-        && MODEL.equals("SM-F936B")) {
-      // This decoder gets stuck if too many frames are rendered without being processed when
-      // tone-mapping HDR10. This value is experimentally determined. See also b/260408846.
-      // TODO(b/260713009): Add API version check after bug is fixed on new API versions.
-      return 12;
-    }
-
-    // Otherwise don't limit the number of frames that can be pending at a time, to maximize
-    // throughput.
-    return UNLIMITED_PENDING_FRAME_COUNT;
-  }
-
   @RequiresApi(29)
   @RequiresApi(29)
   private static final class Api29 {
   private static final class Api29 {
     @DoNotInline
     @DoNotInline

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -360,7 +360,7 @@ import org.checkerframework.dataflow.qual.Pure;
       return true;
       return true;
     }
     }
 
 
-    if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT
+    if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
         && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
         && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
       return false;
       return false;
     }
     }