Provide access to original media timestamps in AudioSink.

* Add `setOutputStreamOffsetUs(long)` method in `AudioSink`.
* Add private methods `setOutputStreamOffsetUs(long)` method in `MediaCodecRenderer` and `DecoderAudioRenderer`.
* Add protected method `onOutputStreamOffsetUs(long)` method in `MediaCodecRenderer`, in which:
  * `MediaCodecRenderer` itself will be no-op for this method.
  * `MediaCodecAudioRenderer` will propagate this value to its `audioSink`.
* Add logics in `DecoderAudioRenderer` to calculate `outputStreamOffsetUs`.

PiperOrigin-RevId: 479265429

library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java

@@ -428,6 +428,14 @@ public interface AudioSink {
   default void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {}
 
   /**
+   * Sets the offset that is added to the media timestamp before it is passed as {@code
+   * presentationTimeUs} in {@link #handleBuffer(ByteBuffer, long, int)}.
+   *
+   * @param outputStreamOffsetUs The output stream offset in microseconds.
+   */
+  default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {}
+
+  /**
    * Enables tunneling, if possible. The sink is reset if tunneling was previously disabled.
    * Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and
    * requires platform API version 21 onwards.

library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java

@@ -118,6 +118,11 @@ public abstract class DecoderAudioRenderer<
    * end of stream signal to indicate that it has output any remaining buffers before we release it.
    */
   private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 2;
+  /**
+   * Generally there is zero or one pending output stream offset. We track more offsets to allow for
+   * pending output streams that have fewer frames than the codec latency.
+   */
+  private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10;
 
   private final EventDispatcher eventDispatcher;
   private final AudioSink audioSink;
@@ -147,6 +152,9 @@ public abstract class DecoderAudioRenderer<
   private boolean allowPositionDiscontinuity;
   private boolean inputStreamEnded;
   private boolean outputStreamEnded;
+  private long outputStreamOffsetUs;
+  private final long[] pendingOutputStreamOffsetsUs;
+  private int pendingOutputStreamOffsetCount;
 
   public DecoderAudioRenderer() {
     this(/* eventHandler= */ null, /* eventListener= */ null);
@@ -206,6 +214,8 @@ public abstract class DecoderAudioRenderer<
     flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
     decoderReinitializationState = REINITIALIZATION_STATE_NONE;
     audioTrackNeedsConfigure = true;
+    setOutputStreamOffsetUs(C.TIME_UNSET);
+    pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
   }
 
   /**
@@ -390,7 +400,7 @@ public abstract class DecoderAudioRenderer<
         audioSink.handleDiscontinuity();
       }
       if (outputBuffer.isFirstSample()) {
-        audioSink.handleDiscontinuity();
+        processFirstSampleOfStream();
       }
     }
 
@@ -436,6 +446,27 @@ public abstract class DecoderAudioRenderer<
     return false;
   }
 
+  private void processFirstSampleOfStream() {
+    audioSink.handleDiscontinuity();
+    if (pendingOutputStreamOffsetCount != 0) {
+      setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
+      pendingOutputStreamOffsetCount--;
+      System.arraycopy(
+          pendingOutputStreamOffsetsUs,
+          /* srcPos= */ 1,
+          pendingOutputStreamOffsetsUs,
+          /* destPos= */ 0,
+          pendingOutputStreamOffsetCount);
+    }
+  }
+
+  private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
+    this.outputStreamOffsetUs = outputStreamOffsetUs;
+    if (outputStreamOffsetUs != C.TIME_UNSET) {
+      audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
+    }
+  }
+
   private boolean feedInputBuffer() throws DecoderException, ExoPlaybackException {
     if (decoder == null
         || decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM
@@ -585,6 +616,7 @@ public abstract class DecoderAudioRenderer<
   protected void onDisabled() {
     inputFormat = null;
     audioTrackNeedsConfigure = true;
+    setOutputStreamOffsetUs(C.TIME_UNSET);
     try {
       setSourceDrmSession(null);
       releaseDecoder();
@@ -599,6 +631,19 @@ public abstract class DecoderAudioRenderer<
       throws ExoPlaybackException {
     super.onStreamChanged(formats, startPositionUs, offsetUs);
     firstStreamSampleRead = false;
+    if (outputStreamOffsetUs == C.TIME_UNSET) {
+      setOutputStreamOffsetUs(offsetUs);
+    } else {
+      if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
+        Log.w(
+            TAG,
+            "Too many stream changes, so dropping offset: "
+                + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
+      } else {
+        pendingOutputStreamOffsetCount++;
+      }
+      pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs;
+    }
   }
 
   @Override

library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java

@@ -143,6 +143,11 @@ public class ForwardingAudioSink implements AudioSink {
   }
 
   @Override
+  public void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
+    sink.setOutputStreamOffsetUs(outputStreamOffsetUs);
+  }
+
+  @Override
   public void enableTunnelingV21() {
     sink.enableTunnelingV21();
   }

library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java

@@ -734,6 +734,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   }
 
   @Override
+  protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
+    audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
+  }
+
+  @Override
   public void handleMessage(@MessageType int messageType, @Nullable Object message)
       throws ExoPlaybackException {
     switch (messageType) {

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -400,7 +400,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
     pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
     outputStreamStartPositionUs = C.TIME_UNSET;
-    outputStreamOffsetUs = C.TIME_UNSET;
+    setOutputStreamOffsetUs(C.TIME_UNSET);
     // MediaCodec outputs audio buffers in native endian:
     // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
     // and code called from MediaCodecAudioRenderer.processOutputBuffer expects this endianness.
@@ -649,7 +649,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     if (this.outputStreamOffsetUs == C.TIME_UNSET) {
       checkState(this.outputStreamStartPositionUs == C.TIME_UNSET);
       this.outputStreamStartPositionUs = startPositionUs;
-      this.outputStreamOffsetUs = offsetUs;
+      setOutputStreamOffsetUs(offsetUs);
     } else {
       if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
         Log.w(
@@ -686,7 +686,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     }
     formatQueue.clear();
     if (pendingOutputStreamOffsetCount != 0) {
-      outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1];
+      setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
       outputStreamStartPositionUs =
           pendingOutputStreamStartPositionsUs[pendingOutputStreamOffsetCount - 1];
       pendingOutputStreamOffsetCount = 0;
@@ -705,7 +705,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   protected void onDisabled() {
     inputFormat = null;
     outputStreamStartPositionUs = C.TIME_UNSET;
-    outputStreamOffsetUs = C.TIME_UNSET;
+    setOutputStreamOffsetUs(C.TIME_UNSET);
     pendingOutputStreamOffsetCount = 0;
     flushOrReleaseCodec();
   }
@@ -1586,7 +1586,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     while (pendingOutputStreamOffsetCount != 0
         && presentationTimeUs >= pendingOutputStreamSwitchTimesUs[0]) {
       outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[0];
-      outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
+      setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
       pendingOutputStreamOffsetCount--;
       System.arraycopy(
           pendingOutputStreamStartPositionsUs,
@@ -1636,6 +1636,17 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
         DISCARD_REASON_REUSE_NOT_IMPLEMENTED);
   }
 
+  /**
+   * Called after the output stream offset changes.
+   *
+   * <p>The default implementation is a no-op.
+   *
+   * @param outputStreamOffsetUs The output stream offset in microseconds.
+   */
+  protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
+    // Do nothing
+  }
+
   @Override
   public boolean isEnded() {
     return outputStreamEnded;
@@ -2044,6 +2055,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     return outputStreamOffsetUs;
   }
 
+  private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
+    this.outputStreamOffsetUs = outputStreamOffsetUs;
+    if (outputStreamOffsetUs != C.TIME_UNSET) {
+      onOutputStreamOffsetUsChanged(outputStreamOffsetUs);
+    }
+  }
+
   /** Returns whether this renderer supports the given {@link Format Format's} DRM scheme. */
   protected static boolean supportsFormatDrm(Format format) {
     return format.cryptoType == C.CRYPTO_TYPE_NONE || format.cryptoType == C.CRYPTO_TYPE_FRAMEWORK;

library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java

@@ -146,7 +146,8 @@ public class DecoderAudioRendererTest {
   }
 
   @Test
-  public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception {
+  public void firstSampleOfStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
+      throws Exception {
     when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
     when(mockAudioSink.isEnded()).thenReturn(true);
     InOrder inOrderAudioSink = inOrder(mockAudioSink);
@@ -177,12 +178,15 @@ public class DecoderAudioRendererTest {
       audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
     }
 
+    inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
     inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
     inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
   }
 
   @Test
-  public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception {
+  public void
+      firstSampleOfReplacementStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
+          throws Exception {
     when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
     when(mockAudioSink.isEnded()).thenReturn(true);
     InOrder inOrderAudioSink = inOrder(mockAudioSink);
@@ -233,9 +237,11 @@ public class DecoderAudioRendererTest {
       audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
     }
 
+    inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
     inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
     inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
     inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
+    inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(1_000_000);
     inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
   }
 

library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java

@@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.robolectric.Shadows.shadowOf;
@@ -57,6 +58,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
@@ -325,6 +327,61 @@ public class MediaCodecAudioRendererTest {
   }
 
   @Test
+  public void render_callsAudioSinkSetOutputStreamOffset_whenReplaceStream() throws Exception {
+    FakeSampleStream fakeSampleStream1 =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            AUDIO_AAC,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
+                oneByteSample(/* timeUs= */ 1_000),
+                END_OF_STREAM_ITEM));
+    fakeSampleStream1.writeData(/* startPositionUs= */ 0);
+    FakeSampleStream fakeSampleStream2 =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            AUDIO_AAC,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME),
+                oneByteSample(/* timeUs= */ 1_001_000),
+                END_OF_STREAM_ITEM));
+    fakeSampleStream2.writeData(/* startPositionUs= */ 0);
+    mediaCodecAudioRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {AUDIO_AAC},
+        fakeSampleStream1,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 0,
+        /* offsetUs= */ 0);
+
+    mediaCodecAudioRenderer.start();
+    while (!mediaCodecAudioRenderer.hasReadStreamToEnd()) {
+      mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+    }
+    mediaCodecAudioRenderer.replaceStream(
+        new Format[] {AUDIO_AAC},
+        fakeSampleStream2,
+        /* startPositionUs= */ 1_000_000,
+        /* offsetUs= */ 1_000_000);
+    mediaCodecAudioRenderer.setCurrentStreamFinal();
+    while (!mediaCodecAudioRenderer.isEnded()) {
+      mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+    }
+
+    InOrder inOrderAudioSink = inOrder(audioSink);
+    inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(0);
+    inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(1_000_000);
+  }
+
+  @Test
   public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception {
     Format mediaFormat =
         new Format.Builder()