Keep audio time in sync with arbitrary decodeOnly samples.

AudioTrack time will go out of sync if the decodeOnly flag
is set of arbitrary samples (as opposed to just those following
a seek). It's a pretty obscure case and it would be weird for
anyone to do it, but we should be robust against it anyway.

library/src/main/java/com/google/android/exoplayer/MediaCodecAudioTrackRenderer.java

@@ -95,6 +95,10 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
   private static final int MIN_PLAYHEAD_OFFSET_SAMPLE_INTERVAL_US = 30000;
   private static final int MIN_TIMESTAMP_SAMPLE_INTERVAL_US = 500000;
 
+  private static final int START_NOT_SET = 0;
+  private static final int START_IN_SYNC = 1;
+  private static final int START_NEED_SYNC = 2;
+
   private final EventListener eventListener;
   private final ConditionVariable audioTrackReleasingConditionVariable;
   private final AudioTimestampCompat audioTimestampCompat;
@@ -119,7 +123,7 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
   private Method audioTrackGetLatencyMethod;
   private int audioSessionId;
   private long submittedBytes;
-  private boolean audioTrackStartMediaTimeSet;
+  private int audioTrackStartMediaTimeState;
   private long audioTrackStartMediaTimeUs;
   private long audioTrackResumeSystemTimeUs;
   private long lastReportedCurrentPositionUs;
@@ -363,7 +367,7 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
       lastRawPlaybackHeadPosition = 0;
       rawPlaybackHeadWrapCount = 0;
       audioTrackStartMediaTimeUs = 0;
-      audioTrackStartMediaTimeSet = false;
+      audioTrackStartMediaTimeState = START_NOT_SET;
       resetSyncParams();
       int playState = audioTrack.getPlayState();
       if (playState == AudioTrack.PLAYSTATE_PLAYING) {
@@ -429,7 +433,7 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
   protected long getCurrentPositionUs() {
     long systemClockUs = System.nanoTime() / 1000;
     long currentPositionUs;
-    if (audioTrack == null || !audioTrackStartMediaTimeSet) {
+    if (audioTrack == null || audioTrackStartMediaTimeState == START_NOT_SET) {
       // The AudioTrack hasn't started.
       currentPositionUs = super.getCurrentPositionUs();
     } else if (audioTimestampSet) {
@@ -461,7 +465,8 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
   }
 
   private void maybeSampleSyncParams() {
-    if (audioTrack == null || !audioTrackStartMediaTimeSet || getState() != STATE_STARTED) {
+    if (audioTrack == null || audioTrackStartMediaTimeState == START_NOT_SET
+        || getState() != STATE_STARTED) {
       // The AudioTrack isn't playing.
       return;
     }
@@ -549,25 +554,40 @@ public class MediaCodecAudioTrackRenderer extends MediaCodecTrackRenderer {
 
   @Override
   protected boolean processOutputBuffer(long timeUs, MediaCodec codec, ByteBuffer buffer,
-      MediaCodec.BufferInfo bufferInfo, int bufferIndex) throws ExoPlaybackException {
+      MediaCodec.BufferInfo bufferInfo, int bufferIndex, boolean shouldSkip)
+      throws ExoPlaybackException {
+    if (shouldSkip) {
+      codec.releaseOutputBuffer(bufferIndex, false);
+      codecCounters.skippedOutputBufferCount++;
+      if (audioTrackStartMediaTimeState == START_IN_SYNC) {
+        // Skipping the sample will push track time out of sync. We'll need to sync again.
+        audioTrackStartMediaTimeState = START_NEED_SYNC;
+      }
+      return true;
+    }
+
     if (temporaryBufferSize == 0) {
       // This is the first time we've seen this {@code buffer}.
-
       // Note: presentationTimeUs corresponds to the end of the sample, not the start.
       long bufferStartTime = bufferInfo.presentationTimeUs -
           framesToDurationUs(bufferInfo.size / frameSize);
-      if (!audioTrackStartMediaTimeSet) {
+      if (audioTrackStartMediaTimeState == START_NOT_SET) {
         audioTrackStartMediaTimeUs = Math.max(0, bufferStartTime);
-        audioTrackStartMediaTimeSet = true;
+        audioTrackStartMediaTimeState = START_IN_SYNC;
       } else {
         // Sanity check that bufferStartTime is consistent with the expected value.
         long expectedBufferStartTime = audioTrackStartMediaTimeUs +
             framesToDurationUs(submittedBytes / frameSize);
-        if (Math.abs(expectedBufferStartTime - bufferStartTime) > 200000) {
+        if (audioTrackStartMediaTimeState == START_IN_SYNC
+            && Math.abs(expectedBufferStartTime - bufferStartTime) > 200000) {
           Log.e(TAG, "Discontinuity detected [expected " + expectedBufferStartTime + ", got " +
               bufferStartTime + "]");
-          // Adjust audioTrackStartMediaTimeUs to compensate for the discontinuity. Also reset
-          // lastReportedCurrentPositionUs to allow time to jump backwards if it really wants to.
+          audioTrackStartMediaTimeState = START_NEED_SYNC;
+        }
+        if (audioTrackStartMediaTimeState == START_NEED_SYNC) {
+          // Adjust audioTrackStartMediaTimeUs to be consistent with the current buffer's start
+          // time and the number of bytes submitted. Also reset lastReportedCurrentPositionUs to
+          // allow time to jump backwards if it really wants to.
           audioTrackStartMediaTimeUs += (bufferStartTime - expectedBufferStartTime);
           lastReportedCurrentPositionUs = 0;
         }

library/src/main/java/com/google/android/exoplayer/MediaCodecTrackRenderer.java

@@ -391,7 +391,9 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
     while (result == SampleSource.SAMPLE_READ && currentPositionUs <= timeUs) {
       result = source.readData(trackIndex, currentPositionUs, formatHolder, sampleHolder, false);
       if (result == SampleSource.SAMPLE_READ) {
-        currentPositionUs = sampleHolder.timeUs;
+        if (!sampleHolder.decodeOnly) {
+          currentPositionUs = sampleHolder.timeUs;
+        }
         codecCounters.discardedSamplesCount++;
       } else if (result == SampleSource.FORMAT_READ) {
         onInputFormatChanged(formatHolder);
@@ -677,15 +679,15 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
       return false;
     }
 
-    if (decodeOnlyPresentationTimestamps.remove(outputBufferInfo.presentationTimeUs)) {
-      codec.releaseOutputBuffer(outputIndex, false);
-      outputIndex = -1;
-      return true;
-    }
-
+    boolean decodeOnly = decodeOnlyPresentationTimestamps.contains(
+        outputBufferInfo.presentationTimeUs);
     if (processOutputBuffer(timeUs, codec, outputBuffers[outputIndex], outputBufferInfo,
-        outputIndex)) {
-      currentPositionUs = outputBufferInfo.presentationTimeUs;
+        outputIndex, decodeOnly)) {
+      if (decodeOnly) {
+        decodeOnlyPresentationTimestamps.remove(outputBufferInfo.presentationTimeUs);
+      } else {
+        currentPositionUs = outputBufferInfo.presentationTimeUs;
+      }
       outputIndex = -1;
       return true;
     }
@@ -701,7 +703,8 @@ public abstract class MediaCodecTrackRenderer extends TrackRenderer {
    * @throws ExoPlaybackException If an error occurs processing the output buffer.
    */
   protected abstract boolean processOutputBuffer(long timeUs, MediaCodec codec, ByteBuffer buffer,
-      MediaCodec.BufferInfo bufferInfo, int bufferIndex) throws ExoPlaybackException;
+      MediaCodec.BufferInfo bufferInfo, int bufferIndex, boolean shouldSkip)
+      throws ExoPlaybackException;
 
   /**
    * Returns the name of the secure variant of a given decoder.

library/src/main/java/com/google/android/exoplayer/MediaCodecVideoTrackRenderer.java

@@ -29,7 +29,7 @@ import android.view.Surface;
 import java.nio.ByteBuffer;
 
 /**
- * Decodes and renders video using {@MediaCodec}.
+ * Decodes and renders video using {@link MediaCodec}.
  */
 @TargetApi(16)
 public class MediaCodecVideoTrackRenderer extends MediaCodecTrackRenderer {
@@ -338,7 +338,12 @@ public class MediaCodecVideoTrackRenderer extends MediaCodecTrackRenderer {
 
   @Override
   protected boolean processOutputBuffer(long timeUs, MediaCodec codec, ByteBuffer buffer,
-      MediaCodec.BufferInfo bufferInfo, int bufferIndex) {
+      MediaCodec.BufferInfo bufferInfo, int bufferIndex, boolean shouldSkip) {
+    if (shouldSkip) {
+      skipOutputBuffer(codec, bufferIndex);
+      return true;
+    }
+
     long earlyUs = bufferInfo.presentationTimeUs - timeUs;
     if (earlyUs < -30000) {
       // We're more than 30ms late rendering the frame.
@@ -371,6 +376,13 @@ public class MediaCodecVideoTrackRenderer extends MediaCodecTrackRenderer {
     return false;
   }
 
+  private void skipOutputBuffer(MediaCodec codec, int bufferIndex) {
+    TraceUtil.beginSection("skipVideoBuffer");
+    codec.releaseOutputBuffer(bufferIndex, false);
+    TraceUtil.endSection();
+    codecCounters.skippedOutputBufferCount++;
+  }
+
   private void dropOutputBuffer(MediaCodec codec, int bufferIndex) {
     TraceUtil.beginSection("dropVideoBuffer");
     codec.releaseOutputBuffer(bufferIndex, false);