Propagate gapless audio delay & padding.

MediaCodec does not need to be re-created in the
event of gapless metadata.

PiperOrigin-RevId: 318439694

RELEASENOTES.md

@@ -170,6 +170,8 @@
         ([#7404](https://github.com/google/ExoPlayer/issues/7404)).
     *   Adjust input timestamps in `MediaCodecRenderer` to account for the
         Codec2 MP3 decoder having lower timestamps on the output side.
+    *   Propagate gapless audio metadata without the need to recreate the audio
+        decoders.
 *   DASH:
     *   Enable support for embedded CEA-708.
 *   HLS:

library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java

@@ -310,16 +310,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   @Override
   protected @KeepCodecResult int canKeepCodec(
       MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) {
-    // TODO: We currently rely on recreating the codec when encoder delay or padding is non-zero.
-    // Re-creating the codec is necessary to guarantee that onOutputMediaFormatChanged is called,
-    // which is where encoder delay and padding are propagated to the sink. We should find a better
-    // way to propagate these values, and then allow the codec to be re-used in cases where this
-    // would otherwise be possible.
-    if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize
-        || oldFormat.encoderDelay != 0
-        || oldFormat.encoderPadding != 0
-        || newFormat.encoderDelay != 0
-        || newFormat.encoderPadding != 0) {
+    if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize) {
       return KEEP_CODEC_RESULT_NO;
     } else if (codecInfo.isSeamlessAdaptationSupported(
         oldFormat, newFormat, /* isNewFormatComplete= */ true)) {
@@ -388,9 +379,14 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   }
 
   @Override
-  protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
-      throws ExoPlaybackException {
+  protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
+    configureOutput(outputFormat);
+  }
+
+  @Override
+  protected void configureOutput(Format outputFormat) throws ExoPlaybackException {
     @C.Encoding int encoding;
+    MediaFormat mediaFormat;
     int channelCount;
     int sampleRate;
     if (passthroughFormat != null) {
@@ -398,18 +394,19 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       channelCount = passthroughFormat.channelCount;
       sampleRate = passthroughFormat.sampleRate;
     } else {
-      if (outputMediaFormat.containsKey(VIVO_BITS_PER_SAMPLE_KEY)) {
-        encoding = Util.getPcmEncoding(outputMediaFormat.getInteger(VIVO_BITS_PER_SAMPLE_KEY));
+      mediaFormat = getCodec().getOutputFormat();
+      if (mediaFormat.containsKey(VIVO_BITS_PER_SAMPLE_KEY)) {
+        encoding = Util.getPcmEncoding(mediaFormat.getInteger(VIVO_BITS_PER_SAMPLE_KEY));
       } else {
-        encoding = getPcmEncoding(inputFormat);
+        encoding = getPcmEncoding(outputFormat);
       }
-      channelCount = outputMediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
-      sampleRate = outputMediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
+      channelCount = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
+      sampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
     }
     @Nullable int[] channelMap = null;
-    if (codecNeedsDiscardChannelsWorkaround && channelCount == 6 && inputFormat.channelCount < 6) {
-      channelMap = new int[inputFormat.channelCount];
-      for (int i = 0; i < inputFormat.channelCount; i++) {
+    if (codecNeedsDiscardChannelsWorkaround && channelCount == 6 && outputFormat.channelCount < 6) {
+      channelMap = new int[outputFormat.channelCount];
+      for (int i = 0; i < outputFormat.channelCount; i++) {
         channelMap[i] = i;
       }
     }
@@ -420,11 +417,10 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
           sampleRate,
           /* specifiedBufferSize= */ 0,
           channelMap,
-          inputFormat.encoderDelay,
-          inputFormat.encoderPadding);
+          outputFormat.encoderDelay,
+          outputFormat.encoderPadding);
     } catch (AudioSink.ConfigurationException e) {
-      // TODO(internal: b/145658993) Use outputFormat instead.
-      throw createRendererException(e, inputFormat);
+      throw createRendererException(e, outputFormat);
     }
   }
 

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -405,6 +405,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private boolean waitingForFirstSampleInFormat;
   private boolean pendingOutputEndOfStream;
   @MediaCodecOperationMode private int mediaCodecOperationMode;
+  @Nullable private ExoPlaybackException pendingPlaybackException;
   protected DecoderCounters decoderCounters;
   private long outputStreamOffsetUs;
   private int pendingOutputStreamOffsetCount;
@@ -623,12 +624,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   }
 
   /**
+   * Sets an exception to be re-thrown by render.
+   *
+   * @param exception The exception.
+   */
+  protected void setPendingPlaybackException(ExoPlaybackException exception) {
+    pendingPlaybackException = exception;
+  }
+
+  /**
    * Polls the pending output format queue for a given buffer timestamp. If a format is present, it
    * is removed and returned. Otherwise returns {@code null}. Subclasses should only call this
    * method if they are taking over responsibility for output format propagation (e.g., when using
    * video tunneling).
+   *
+   * @throws ExoPlaybackException Thrown if an error occurs as a result of the output format change.
    */
-  protected final void updateOutputFormatForTime(long presentationTimeUs) {
+  protected final void updateOutputFormatForTime(long presentationTimeUs)
+      throws ExoPlaybackException {
     @Nullable Format format = formatQueue.pollFloor(presentationTimeUs);
     if (format != null) {
       outputFormat = format;
@@ -784,6 +797,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       pendingOutputEndOfStream = false;
       processEndOfStream();
     }
+    if (pendingPlaybackException != null) {
+      ExoPlaybackException playbackException = pendingPlaybackException;
+      pendingPlaybackException = null;
+      throw playbackException;
+    }
+
     try {
       if (outputStreamEnded) {
         renderToEndOfStream();
@@ -908,6 +927,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     codecInfo = null;
     codecFormat = null;
     codecHasOutputMediaFormat = false;
+    pendingPlaybackException = null;
     codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
     codecAdaptationWorkaroundMode = ADAPTATION_WORKAROUND_MODE_NEVER;
     codecNeedsReconfigureWorkaround = false;
@@ -1490,8 +1510,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * <p>The default implementation is a no-op.
    *
    * @param outputFormat The new output {@link Format}.
+   * @throws ExoPlaybackException Thrown if an error occurs handling the new output format.
    */
-  protected void onOutputFormatChanged(Format outputFormat) {
+  protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
     // Do nothing.
   }
 
@@ -1501,8 +1522,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * <p>The default implementation is a no-op.
    *
    * @param outputFormat The format to configure the output with.
+   * @throws ExoPlaybackException Thrown if an error occurs configuring the output.
    */
-  protected void configureOutput(Format outputFormat) {
+  protected void configureOutput(Format outputFormat) throws ExoPlaybackException {
     // Do nothing.
   }
 
@@ -1538,8 +1560,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * <p>The default implementation is a no-op.
    *
    * @param buffer The buffer to be queued.
+   * @throws ExoPlaybackException Thrown if an error occurs handling the input buffer.
    */
-  protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
+  protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
     // Do nothing.
   }
 

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -643,11 +643,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   /**
    * Called immediately before an input buffer is queued into the codec.
    *
+   * <p>In tunneling mode for pre Marshmallow, the buffer is treated as if immediately output.
+   *
    * @param buffer The buffer to be queued.
+   * @throws ExoPlaybackException Thrown if an error occurs handling the input buffer.
    */
   @CallSuper
   @Override
-  protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
+  protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
     // In tunneling mode the device may do frame rate conversion, so in general we can't keep track
     // of the number of buffers in the codec.
     if (!tunneling) {
@@ -891,7 +894,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
 
   /** Called when a buffer was processed in tunneling mode. */
-  protected void onProcessedTunneledBuffer(long presentationTimeUs) {
+  protected void onProcessedTunneledBuffer(long presentationTimeUs) throws ExoPlaybackException {
     updateOutputFormatForTime(presentationTimeUs);
     maybeNotifyVideoSizeChanged();
     decoderCounters.renderedOutputBufferCount++;
@@ -1808,7 +1811,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       if (presentationTimeUs == TUNNELING_EOS_PRESENTATION_TIME_US) {
         onProcessedTunneledEndOfStream();
       } else {
-        onProcessedTunneledBuffer(presentationTimeUs);
+        try {
+          onProcessedTunneledBuffer(presentationTimeUs);
+        } catch (ExoPlaybackException e) {
+          setPendingPlaybackException(e);
+        }
       }
     }
   }

library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.audio;
 
+import static org.junit.Assert.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
@@ -25,13 +26,12 @@ import android.os.SystemClock;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.ExoPlaybackException;
 import com.google.android.exoplayer2.Format;
-import com.google.android.exoplayer2.RendererCapabilities;
 import com.google.android.exoplayer2.RendererConfiguration;
 import com.google.android.exoplayer2.drm.DrmSessionManager;
 import com.google.android.exoplayer2.mediacodec.MediaCodecInfo;
 import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
-import com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryException;
 import com.google.android.exoplayer2.testutil.FakeSampleStream;
 import com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem;
 import com.google.android.exoplayer2.util.MimeTypes;
@@ -61,6 +61,7 @@ public class MediaCodecAudioRendererTest {
           .build();
 
   private MediaCodecAudioRenderer mediaCodecAudioRenderer;
+  private MediaCodecSelector mediaCodecSelector;
 
   @Mock private AudioSink audioSink;
 
@@ -72,7 +73,7 @@ public class MediaCodecAudioRendererTest {
 
     when(audioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
 
-    MediaCodecSelector mediaCodecSelector =
+    mediaCodecSelector =
         new MediaCodecSelector() {
           @Override
           public List<MediaCodecInfo> getDecoderInfos(
@@ -98,17 +99,75 @@ public class MediaCodecAudioRendererTest {
             /* enableDecoderFallback= */ false,
             /* eventHandler= */ null,
             /* eventListener= */ null,
-            audioSink) {
-          @Override
-          protected int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format)
-              throws DecoderQueryException {
-            return RendererCapabilities.create(FORMAT_HANDLED);
-          }
-        };
+            audioSink);
+  }
+
+  @Test
+  public void render_configuresAudioSink_afterFormatChange() throws Exception {
+    Format changedFormat = AUDIO_AAC.buildUpon().setSampleRate(48_000).setEncoderDelay(400).build();
+
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            /* format= */ AUDIO_AAC,
+            DrmSessionManager.DUMMY,
+            /* eventDispatcher= */ null,
+            /* firstSampleTimeUs= */ 0,
+            /* timeUsIncrement= */ 50,
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(changedFormat),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            FakeSampleStreamItem.END_OF_STREAM_ITEM);
+
+    mediaCodecAudioRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {AUDIO_AAC, changedFormat},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ false,
+        /* offsetUs */ 0);
+
+    mediaCodecAudioRenderer.start();
+    mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
+    mediaCodecAudioRenderer.render(/* positionUs= */ 250, SystemClock.elapsedRealtime() * 1000);
+    mediaCodecAudioRenderer.setCurrentStreamFinal();
+
+    int positionUs = 500;
+    do {
+      mediaCodecAudioRenderer.render(positionUs, SystemClock.elapsedRealtime() * 1000);
+      positionUs += 250;
+    } while (!mediaCodecAudioRenderer.isEnded());
+
+    verify(audioSink)
+        .configure(
+            AUDIO_AAC.pcmEncoding,
+            AUDIO_AAC.channelCount,
+            AUDIO_AAC.sampleRate,
+            /* specifiedBufferSize= */ 0,
+            /* outputChannels= */ null,
+            AUDIO_AAC.encoderDelay,
+            AUDIO_AAC.encoderPadding);
+
+    verify(audioSink)
+        .configure(
+            changedFormat.pcmEncoding,
+            changedFormat.channelCount,
+            changedFormat.sampleRate,
+            /* specifiedBufferSize= */ 0,
+            /* outputChannels= */ null,
+            changedFormat.encoderDelay,
+            changedFormat.encoderPadding);
   }
 
   @Test
-  public void render_configuresAudioSink() throws Exception {
+  public void render_configuresAudioSink_afterGaplessFormatChange() throws Exception {
+    Format changedFormat =
+        AUDIO_AAC.buildUpon().setEncoderDelay(400).setEncoderPadding(232).build();
+
     FakeSampleStream fakeSampleStream =
         new FakeSampleStream(
             /* format= */ AUDIO_AAC,
@@ -117,11 +176,17 @@ public class MediaCodecAudioRendererTest {
             /* firstSampleTimeUs= */ 0,
             /* timeUsIncrement= */ 50,
             new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(changedFormat),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
             FakeSampleStreamItem.END_OF_STREAM_ITEM);
 
     mediaCodecAudioRenderer.enable(
         RendererConfiguration.DEFAULT,
-        new Format[] {AUDIO_AAC},
+        new Format[] {AUDIO_AAC, changedFormat},
         fakeSampleStream,
         /* positionUs= */ 0,
         /* joining= */ false,
@@ -148,5 +213,77 @@ public class MediaCodecAudioRendererTest {
             /* outputChannels= */ null,
             AUDIO_AAC.encoderDelay,
             AUDIO_AAC.encoderPadding);
+
+    verify(audioSink)
+        .configure(
+            changedFormat.pcmEncoding,
+            changedFormat.channelCount,
+            changedFormat.sampleRate,
+            /* specifiedBufferSize= */ 0,
+            /* outputChannels= */ null,
+            changedFormat.encoderDelay,
+            changedFormat.encoderPadding);
+  }
+
+  @Test
+  public void render_throwsExoPlaybackExceptionJustOnce_whenSet() throws Exception {
+    MediaCodecAudioRenderer exceptionThrowingRenderer =
+        new MediaCodecAudioRenderer(
+            ApplicationProvider.getApplicationContext(),
+            mediaCodecSelector,
+            /* eventHandler= */ null,
+            /* eventListener= */ null) {
+          @Override
+          protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
+            super.onOutputFormatChanged(outputFormat);
+            if (!outputFormat.equals(AUDIO_AAC)) {
+              setPendingPlaybackException(
+                  ExoPlaybackException.createForRenderer(
+                      new AudioSink.ConfigurationException("Test"),
+                      "rendererName",
+                      /* rendererIndex= */ 0,
+                      outputFormat,
+                      FORMAT_HANDLED));
+            }
+          }
+        };
+
+    Format changedFormat = AUDIO_AAC.buildUpon().setSampleRate(32_000).build();
+
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            /* format= */ AUDIO_AAC,
+            DrmSessionManager.DUMMY,
+            /* eventDispatcher= */ null,
+            /* firstSampleTimeUs= */ 0,
+            /* timeUsIncrement= */ 50,
+            new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
+            FakeSampleStreamItem.END_OF_STREAM_ITEM);
+
+    exceptionThrowingRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {AUDIO_AAC, changedFormat},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ false,
+        /* offsetUs */ 0);
+
+    exceptionThrowingRenderer.start();
+    exceptionThrowingRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
+    exceptionThrowingRenderer.render(/* positionUs= */ 250, SystemClock.elapsedRealtime() * 1000);
+
+    // Simulating the exception being thrown when not traceable back to render.
+    exceptionThrowingRenderer.onOutputFormatChanged(changedFormat);
+
+    assertThrows(
+        ExoPlaybackException.class,
+        () ->
+            exceptionThrowingRenderer.render(
+                /* positionUs= */ 500, SystemClock.elapsedRealtime() * 1000));
+
+    // Doesn't throw an exception because it's cleared after being thrown in the previous call to
+    // render.
+    exceptionThrowingRenderer.render(/* positionUs= */ 750, SystemClock.elapsedRealtime() * 1000);
   }
 }

playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DebugRenderersFactory.java

@@ -142,7 +142,7 @@ import java.util.ArrayList;
     }
 
     @Override
-    protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
+    protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
       super.onQueueInputBuffer(buffer);
       insertTimestamp(buffer.timeUs);
       maybeShiftTimestampsList();