Adjust input timestamps for the Codec2 MP3 decoder.

Output timestamps are calculated by the codec based on the buffers,
which is offset in Codec2. This adjusts the input timestamps as they
are passed in so they will match the output timestamps produced by
the MediaCodec.

PiperOrigin-RevId: 314963830

RELEASENOTES.md

@@ -152,6 +152,8 @@
     *   Check `DefaultAudioSink` supports passthrough, in addition to checking
     *   Check `DefaultAudioSink` supports passthrough, in addition to checking
         the `AudioCapabilities`
         the `AudioCapabilities`
         ([#7404](https://github.com/google/ExoPlayer/issues/7404)).
         ([#7404](https://github.com/google/ExoPlayer/issues/7404)).
+    *   Adjust input timestamps in `MediaCodecRenderer` to account for the
+        Codec2 MP3 decoder having lower timestamps on the output side.
 *   DASH:
 *   DASH:
     *   Enable support for embedded CEA-708.
     *   Enable support for embedded CEA-708.
 *   HLS:
 *   HLS:

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/C2Mp3TimestampTracker.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.mediacodec;
+
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.audio.MpegAudioUtil;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.Log;
+
+/**
+ * Tracks the number of processed samples to calculate an accurate current timestamp, matching the
+ * calculations made in the Codec2 Mp3 decoder.
+ */
+/* package */ final class C2Mp3TimestampTracker {
+
+  // Mirroring the actual codec, as can be found at
+  // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/codec2/components/mp3/C2SoftMp3Dec.h;l=55;drc=3665390c9d32a917398b240c5a46ced07a3b65eb
+  private static final long DECODER_DELAY_SAMPLES = 529;
+  private static final String TAG = "C2Mp3TimestampTracker";
+
+  private long processedSamples;
+  private long anchorTimestampUs;
+  private boolean audioHeaderInvalid;
+
+  /**
+   * Resets the timestamp tracker.
+   *
+   * <p>This should be done when the codec is flushed.
+   */
+  public void reset() {
+    processedSamples = 0;
+    anchorTimestampUs = 0;
+    audioHeaderInvalid = false;
+  }
+
+  /**
+   * Updates the tracker with the given input buffer and returns the expected output timestamp.
+   *
+   * @param format The format associated with the buffer.
+   * @param buffer The current input buffer.
+   * @return The expected output presentation time, in microseconds.
+   */
+  public long updateAndGetPresentationTimeUs(Format format, DecoderInputBuffer buffer) {
+    if (audioHeaderInvalid || buffer.data == null) {
+      return buffer.timeUs;
+    }
+
+    // These calculations mirror the timestamp calculations in the Codec2 Mp3 Decoder.
+    // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/codec2/components/mp3/C2SoftMp3Dec.cpp;l=464;drc=ed134640332fea70ca4b05694289d91a5265bb46
+    long presentationTimeUs = processedSamples * C.MICROS_PER_SECOND / format.sampleRate;
+    int sampleHeaderData = 0;
+    for (int i = 0; i < 4; i++) {
+      sampleHeaderData <<= 8;
+      sampleHeaderData |= buffer.data.get(i) & 0xFF;
+    }
+
+    int frameCount = MpegAudioUtil.parseMpegAudioFrameSampleCount(sampleHeaderData);
+    if (frameCount == C.LENGTH_UNSET) {
+      Log.w(TAG, "MPEG audio header is invalid.");
+      return buffer.timeUs;
+    }
+    long outSize = frameCount * format.channelCount * 2L;
+    boolean isFirstSample = processedSamples == 0;
+    long outOffset = 0;
+    if (isFirstSample) {
+      anchorTimestampUs = buffer.timeUs;
+      outOffset = DECODER_DELAY_SAMPLES;
+    }
+    processedSamples += (outSize / (format.channelCount * 2L)) - outOffset;
+    return anchorTimestampUs + presentationTimeUs;
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -395,6 +395,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private boolean codecNeedsAdaptationWorkaroundBuffer;
   private boolean codecNeedsAdaptationWorkaroundBuffer;
   private boolean shouldSkipAdaptationWorkaroundOutputBuffer;
   private boolean shouldSkipAdaptationWorkaroundOutputBuffer;
   private boolean codecNeedsEosPropagation;
   private boolean codecNeedsEosPropagation;
+  @Nullable private C2Mp3TimestampTracker c2Mp3TimestampTracker;
   private ByteBuffer[] inputBuffers;
   private ByteBuffer[] inputBuffers;
   private ByteBuffer[] outputBuffers;
   private ByteBuffer[] outputBuffers;
   private long codecHotswapDeadlineMs;
   private long codecHotswapDeadlineMs;
@@ -911,6 +912,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     decodeOnlyPresentationTimestamps.clear();
     decodeOnlyPresentationTimestamps.clear();
     largestQueuedPresentationTimeUs = C.TIME_UNSET;
     largestQueuedPresentationTimeUs = C.TIME_UNSET;
     lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
     lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
+    if (c2Mp3TimestampTracker != null) {
+      c2Mp3TimestampTracker.reset();
+    }
     codecDrainState = DRAIN_STATE_NONE;
     codecDrainState = DRAIN_STATE_NONE;
     codecDrainAction = DRAIN_ACTION_NONE;
     codecDrainAction = DRAIN_ACTION_NONE;
     // Reconfiguration data sent shortly before the flush may not have been processed by the
     // Reconfiguration data sent shortly before the flush may not have been processed by the
@@ -944,6 +948,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     codecNeedsEosOutputExceptionWorkaround = false;
     codecNeedsEosOutputExceptionWorkaround = false;
     codecNeedsMonoChannelCountWorkaround = false;
     codecNeedsMonoChannelCountWorkaround = false;
     codecNeedsEosPropagation = false;
     codecNeedsEosPropagation = false;
+    c2Mp3TimestampTracker = null;
     codecReconfigured = false;
     codecReconfigured = false;
     codecReconfigurationState = RECONFIGURATION_STATE_NONE;
     codecReconfigurationState = RECONFIGURATION_STATE_NONE;
     resetCodecBuffers();
     resetCodecBuffers();
@@ -1157,6 +1162,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
         codecNeedsMonoChannelCountWorkaround(codecName, codecFormat);
         codecNeedsMonoChannelCountWorkaround(codecName, codecFormat);
     codecNeedsEosPropagation =
     codecNeedsEosPropagation =
         codecNeedsEosPropagationWorkaround(codecInfo) || getCodecNeedsEosPropagation();
         codecNeedsEosPropagationWorkaround(codecInfo) || getCodecNeedsEosPropagation();
+    if ("c2.android.mp3.decoder".equals(codecInfo.name)) {
+      c2Mp3TimestampTracker = new C2Mp3TimestampTracker();
+    }
+
     if (getState() == STATE_STARTED) {
     if (getState() == STATE_STARTED) {
       codecHotswapDeadlineMs = SystemClock.elapsedRealtime() + MAX_CODEC_HOTSWAP_TIME_MS;
       codecHotswapDeadlineMs = SystemClock.elapsedRealtime() + MAX_CODEC_HOTSWAP_TIME_MS;
     }
     }
@@ -1369,6 +1378,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     }
     }
 
 
     long presentationTimeUs = buffer.timeUs;
     long presentationTimeUs = buffer.timeUs;
+
+    if (c2Mp3TimestampTracker != null) {
+      presentationTimeUs =
+          c2Mp3TimestampTracker.updateAndGetPresentationTimeUs(inputFormat, buffer);
+    }
+
     if (buffer.isDecodeOnly()) {
     if (buffer.isDecodeOnly()) {
       decodeOnlyPresentationTimestamps.add(presentationTimeUs);
       decodeOnlyPresentationTimestamps.add(presentationTimeUs);
     }
     }

library/core/src/test/java/com/google/android/exoplayer2/mediacodec/C2Mp3TimestampTrackerTest.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.mediacodec;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.MimeTypes;
+import java.nio.ByteBuffer;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link C2Mp3TimestampTracker}. */
+@RunWith(AndroidJUnit4.class)
+public final class C2Mp3TimestampTrackerTest {
+
+  private static final Format AUDIO_MP3 =
+      new Format.Builder()
+          .setSampleMimeType(MimeTypes.AUDIO_MPEG)
+          .setPcmEncoding(C.ENCODING_PCM_16BIT)
+          .setChannelCount(2)
+          .setSampleRate(44_100)
+          .build();
+
+  private DecoderInputBuffer buffer;
+  private C2Mp3TimestampTracker timestampTracker;
+
+  @Before
+  public void setUp() {
+    buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
+    timestampTracker = new C2Mp3TimestampTracker();
+    buffer.data = ByteBuffer.wrap(new byte[] {-1, -5, -24, 60});
+    buffer.timeUs = 100_000;
+  }
+
+  @Test
+  public void whenUpdateCalledMultipleTimes_timestampsIncrease() {
+    long first = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+    long second = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+    long third = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+
+    assertThat(second).isGreaterThan(first);
+    assertThat(third).isGreaterThan(second);
+  }
+
+  @Test
+  public void whenResetCalled_timestampsDecrease() {
+    long first = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+    long second = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+    timestampTracker.reset();
+    long third = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+
+    assertThat(second).isGreaterThan(first);
+    assertThat(third).isLessThan(second);
+  }
+
+  @Test
+  public void whenBufferTimeIsNotZero_firstSampleIsOffset() {
+    long first = timestampTracker.updateAndGetPresentationTimeUs(AUDIO_MP3, buffer);
+
+    assertThat(first).isEqualTo(buffer.timeUs);
+  }
+}