Catch up video rendering by dropping to keyframes

If the current output buffer is very late and the playback position is in a
later group of pictures, drop all buffers to the keyframe preceding the
playback position.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=170975259

library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java

@@ -65,6 +65,14 @@ public final class DecoderCounters {
    * Skipped output buffers are ignored for the purposes of calculating this value.
    * Skipped output buffers are ignored for the purposes of calculating this value.
    */
    */
   public int maxConsecutiveDroppedOutputBufferCount;
   public int maxConsecutiveDroppedOutputBufferCount;
+  /**
+   * The number of times all buffers to a keyframe were dropped.
+   * <p>
+   * Each time buffers to a keyframe are dropped, this counter is increased by one, and the dropped
+   * output buffer counters are increased by one (for the current output buffer) plus the number of
+   * buffers dropped from the source to advance to the keyframe.
+   */
+  public int droppedToKeyframeCount;
 
 
   /**
   /**
    * Should be called to ensure counter values are made visible across threads. The playback thread
    * Should be called to ensure counter values are made visible across threads. The playback thread
@@ -91,6 +99,7 @@ public final class DecoderCounters {
     droppedOutputBufferCount += other.droppedOutputBufferCount;
     droppedOutputBufferCount += other.droppedOutputBufferCount;
     maxConsecutiveDroppedOutputBufferCount = Math.max(maxConsecutiveDroppedOutputBufferCount,
     maxConsecutiveDroppedOutputBufferCount = Math.max(maxConsecutiveDroppedOutputBufferCount,
         other.maxConsecutiveDroppedOutputBufferCount);
         other.maxConsecutiveDroppedOutputBufferCount);
+    droppedToKeyframeCount += other.droppedToKeyframeCount;
   }
   }
 
 
 }
 }

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -25,6 +25,7 @@ import android.media.MediaCrypto;
 import android.media.MediaFormat;
 import android.media.MediaFormat;
 import android.os.Handler;
 import android.os.Handler;
 import android.os.SystemClock;
 import android.os.SystemClock;
+import android.support.annotation.CallSuper;
 import android.support.annotation.NonNull;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
 import android.support.annotation.Nullable;
 import android.util.Log;
 import android.util.Log;
@@ -85,10 +86,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   @C.VideoScalingMode
   @C.VideoScalingMode
   private int scalingMode;
   private int scalingMode;
   private boolean renderedFirstFrame;
   private boolean renderedFirstFrame;
+  private boolean forceRenderFrame;
   private long joiningDeadlineMs;
   private long joiningDeadlineMs;
   private long droppedFrameAccumulationStartTimeMs;
   private long droppedFrameAccumulationStartTimeMs;
   private int droppedFrames;
   private int droppedFrames;
   private int consecutiveDroppedFrameCount;
   private int consecutiveDroppedFrameCount;
+  private int buffersInCodecCount;
 
 
   private int pendingRotationDegrees;
   private int pendingRotationDegrees;
   private float pendingPixelWidthHeightRatio;
   private float pendingPixelWidthHeightRatio;
@@ -414,11 +417,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     }
     }
   }
   }
 
 
+  @CallSuper
   @Override
   @Override
   protected void releaseCodec() {
   protected void releaseCodec() {
     try {
     try {
       super.releaseCodec();
       super.releaseCodec();
     } finally {
     } finally {
+      buffersInCodecCount = 0;
+      forceRenderFrame = false;
       if (dummySurface != null) {
       if (dummySurface != null) {
         if (surface == dummySurface) {
         if (surface == dummySurface) {
           surface = null;
           surface = null;
@@ -429,6 +435,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     }
     }
   }
   }
 
 
+  @CallSuper
+  @Override
+  protected void flushCodec() throws ExoPlaybackException {
+    super.flushCodec();
+    buffersInCodecCount = 0;
+    forceRenderFrame = false;
+  }
+
   @Override
   @Override
   protected void onCodecInitialized(String name, long initializedTimestampMs,
   protected void onCodecInitialized(String name, long initializedTimestampMs,
       long initializationDurationMs) {
       long initializationDurationMs) {
@@ -444,8 +458,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     pendingRotationDegrees = getRotationDegrees(newFormat);
     pendingRotationDegrees = getRotationDegrees(newFormat);
   }
   }
 
 
+  /**
+   * Called immediately before an input buffer is queued into the codec.
+   *
+   * @param buffer The buffer to be queued.
+   */
+  @CallSuper
   @Override
   @Override
   protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
   protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
+    buffersInCodecCount++;
     if (Util.SDK_INT < 23 && tunneling) {
     if (Util.SDK_INT < 23 && tunneling) {
       maybeNotifyRenderedFirstFrame();
       maybeNotifyRenderedFirstFrame();
     }
     }
@@ -492,7 +513,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   @Override
   @Override
   protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
   protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
       ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
       ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
-      boolean shouldSkip) {
+      boolean shouldSkip) throws ExoPlaybackException {
     while (pendingOutputStreamOffsetCount != 0
     while (pendingOutputStreamOffsetCount != 0
         && bufferPresentationTimeUs >= pendingOutputStreamOffsetsUs[0]) {
         && bufferPresentationTimeUs >= pendingOutputStreamOffsetsUs[0]) {
       outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
       outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
@@ -517,7 +538,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       return false;
       return false;
     }
     }
 
 
-    if (!renderedFirstFrame) {
+    if (!renderedFirstFrame || forceRenderFrame) {
+      forceRenderFrame = false;
       if (Util.SDK_INT >= 21) {
       if (Util.SDK_INT >= 21) {
         renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime());
         renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime());
       } else {
       } else {
@@ -544,7 +566,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
         bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs);
         bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs);
     earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;
     earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;
 
 
-    if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) {
+    if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs)
+        && maybeDropBuffersToKeyframe(codec, bufferIndex, presentationTimeUs, positionUs)) {
+      forceRenderFrame = true;
+      return true;
+    } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) {
       dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
       dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
       return true;
       return true;
     }
     }
@@ -578,6 +604,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
   }
 
 
   /**
   /**
+   * Called when an output buffer is successfully processed.
+   *
+   * @param presentationTimeUs The timestamp associated with the output buffer.
+   */
+  @CallSuper
+  @Override
+  protected void onProcessedOutputBuffer(long presentationTimeUs) {
+    buffersInCodecCount--;
+  }
+
+  /**
    * Returns whether the buffer being processed should be dropped.
    * Returns whether the buffer being processed should be dropped.
    *
    *
    * @param earlyUs The time until the buffer should be presented in microseconds. A negative value
    * @param earlyUs The time until the buffer should be presented in microseconds. A negative value
@@ -590,6 +627,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
   }
 
 
   /**
   /**
+   * Returns whether to drop all buffers from the buffer being processed to the keyframe at or after
+   * the current playback position, if possible.
+   *
+   * @param earlyUs The time until the current buffer should be presented in microseconds. A
+   *     negative value indicates that the buffer is late.
+   * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+   *     measured at the start of the current iteration of the rendering loop.
+   */
+  protected boolean shouldDropBuffersToKeyframe(long earlyUs, long elapsedRealtimeUs) {
+    return isBufferVeryLate(earlyUs);
+  }
+
+  /**
    * Skips the output buffer with the specified index.
    * Skips the output buffer with the specified index.
    *
    *
    * @param codec The codec that owns the output buffer.
    * @param codec The codec that owns the output buffer.
@@ -614,12 +664,48 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     TraceUtil.beginSection("dropVideoBuffer");
     TraceUtil.beginSection("dropVideoBuffer");
     codec.releaseOutputBuffer(index, false);
     codec.releaseOutputBuffer(index, false);
     TraceUtil.endSection();
     TraceUtil.endSection();
-    decoderCounters.droppedOutputBufferCount++;
-    droppedFrames++;
-    consecutiveDroppedFrameCount++;
+    updateDroppedBufferCounters(1);
+  }
+
+  /**
+   * Drops frames from the current output buffer to the next keyframe at or before the playback
+   * position. If no such keyframe exists, as the playback position is inside the same group of
+   * pictures as the buffer being processed, returns {@code false}. Returns {@code true} otherwise.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param index The index of the output buffer to drop.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   * @param positionUs The current playback position, in microseconds.
+   * @return Whether any buffers were dropped.
+   * @throws ExoPlaybackException If an error occurs flushing the codec.
+   */
+  protected boolean maybeDropBuffersToKeyframe(MediaCodec codec, int index, long presentationTimeUs,
+      long positionUs) throws ExoPlaybackException {
+    int droppedSourceBufferCount = skipSource(positionUs);
+    if (droppedSourceBufferCount == 0) {
+      return false;
+    }
+    decoderCounters.droppedToKeyframeCount++;
+    // We dropped some buffers to catch up, so update the decoder counters and flush the codec,
+    // which releases all pending buffers buffers including the current output buffer.
+    updateDroppedBufferCounters(buffersInCodecCount + droppedSourceBufferCount);
+    flushCodec();
+    return true;
+  }
+
+  /**
+   * Updates decoder counters to reflect that {@code droppedBufferCount} additional buffers were
+   * dropped.
+   *
+   * @param droppedBufferCount The number of additional dropped buffers.
+   */
+  protected void updateDroppedBufferCounters(int droppedBufferCount) {
+    decoderCounters.droppedOutputBufferCount += droppedBufferCount;
+    droppedFrames += droppedBufferCount;
+    consecutiveDroppedFrameCount += droppedBufferCount;
     decoderCounters.maxConsecutiveDroppedOutputBufferCount = Math.max(consecutiveDroppedFrameCount,
     decoderCounters.maxConsecutiveDroppedOutputBufferCount = Math.max(consecutiveDroppedFrameCount,
         decoderCounters.maxConsecutiveDroppedOutputBufferCount);
         decoderCounters.maxConsecutiveDroppedOutputBufferCount);
-    if (droppedFrames == maxDroppedFramesToNotify) {
+    if (droppedFrames >= maxDroppedFramesToNotify) {
       maybeNotifyDroppedFrames();
       maybeNotifyDroppedFrames();
     }
     }
   }
   }
@@ -740,10 +826,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
   }
 
 
   private static boolean isBufferLate(long earlyUs) {
   private static boolean isBufferLate(long earlyUs) {
-    // Class a buffer as late if it should have been presented more than 30ms ago.
+    // Class a buffer as late if it should have been presented more than 30 ms ago.
     return earlyUs < -30000;
     return earlyUs < -30000;
   }
   }
 
 
+  private static boolean isBufferVeryLate(long earlyUs) {
+    // Class a buffer as very late if it should have been presented more than 500 ms ago.
+    return earlyUs < -500000;
+  }
+
   @TargetApi(23)
   @TargetApi(23)
   private static void setOutputSurfaceV23(MediaCodec codec, Surface surface) {
   private static void setOutputSurfaceV23(MediaCodec codec, Surface surface) {
     codec.setOutputSurface(surface);
     codec.setOutputSurface(surface);

library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java

@@ -155,7 +155,8 @@ public final class DebugTextViewHelper extends Player.DefaultEventListener imple
         + " sb:" + counters.skippedOutputBufferCount
         + " sb:" + counters.skippedOutputBufferCount
         + " rb:" + counters.renderedOutputBufferCount
         + " rb:" + counters.renderedOutputBufferCount
         + " db:" + counters.droppedOutputBufferCount
         + " db:" + counters.droppedOutputBufferCount
-        + " mcdb:" + counters.maxConsecutiveDroppedOutputBufferCount;
+        + " mcdb:" + counters.maxConsecutiveDroppedOutputBufferCount
+        + " dk:" + counters.droppedToKeyframeCount;
   }
   }
 
 
   private static String getPixelAspectRatioString(float pixelAspectRatio) {
   private static String getPixelAspectRatioString(float pixelAspectRatio) {

testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java

@@ -124,7 +124,7 @@ public class DebugRenderersFactory extends DefaultRenderersFactory {
     @Override
     @Override
     protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
     protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
         ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
         ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
-        boolean shouldSkip) {
+        boolean shouldSkip) throws ExoPlaybackException {
       if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) {
       if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) {
         // After the codec has been initialized, don't render the first frame until we've caught up
         // After the codec has been initialized, don't render the first frame until we've caught up
         // to the playback position. Else test runs on devices that do not support dummy surface
         // to the playback position. Else test runs on devices that do not support dummy surface