Provide video frame timestamps to subclasses

Expose the stream offset to BaseRenderer subclasses.

Issue: #2267

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=156837514

library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java

@@ -96,7 +96,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
     this.stream = stream;
     this.stream = stream;
     readEndOfStream = false;
     readEndOfStream = false;
     streamOffsetUs = offsetUs;
     streamOffsetUs = offsetUs;
-    onStreamChanged(formats);
+    onStreamChanged(formats, offsetUs);
   }
   }
 
 
   @Override
   @Override
@@ -183,16 +183,19 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
    * The default implementation is a no-op.
    * The default implementation is a no-op.
    *
    *
    * @param formats The enabled formats.
    * @param formats The enabled formats.
+   * @param offsetUs The offset that will be added to the timestamps of buffers read via
+   *     {@link #readSource(FormatHolder, DecoderInputBuffer, boolean)} so that decoder input
+   *     buffers have monotonically increasing timestamps.
    * @throws ExoPlaybackException If an error occurs.
    * @throws ExoPlaybackException If an error occurs.
    */
    */
-  protected void onStreamChanged(Format[] formats) throws ExoPlaybackException {
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
     // Do nothing.
     // Do nothing.
   }
   }
 
 
   /**
   /**
    * Called when the position is reset. This occurs when the renderer is enabled after
    * Called when the position is reset. This occurs when the renderer is enabled after
-   * {@link #onStreamChanged(Format[])} has been called, and also when a position discontinuity
-   * is encountered.
+   * {@link #onStreamChanged(Format[], long)} has been called, and also when a position
+   * discontinuity is encountered.
    * <p>
    * <p>
    * After a position reset, the renderer's {@link SampleStream} is guaranteed to provide samples
    * After a position reset, the renderer's {@link SampleStream} is guaranteed to provide samples
    * starting from a key frame.
    * starting from a key frame.

library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java

@@ -104,7 +104,7 @@ public final class MetadataRenderer extends BaseRenderer implements Callback {
   }
   }
 
 
   @Override
   @Override
-  protected void onStreamChanged(Format[] formats) throws ExoPlaybackException {
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
     decoder = decoderFactory.createDecoder(formats[0]);
     decoder = decoderFactory.createDecoder(formats[0]);
   }
   }
 
 

library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java

@@ -130,7 +130,7 @@ public final class TextRenderer extends BaseRenderer implements Callback {
   }
   }
 
 
   @Override
   @Override
-  protected void onStreamChanged(Format[] formats) throws ExoPlaybackException {
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
     streamFormat = formats[0];
     streamFormat = formats[0];
     if (decoder != null) {
     if (decoder != null) {
       decoderReplacementState = REPLACEMENT_STATE_SIGNAL_END_OF_STREAM;
       decoderReplacementState = REPLACEMENT_STATE_SIGNAL_END_OF_STREAM;

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -62,11 +62,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private static final int[] STANDARD_LONG_EDGE_VIDEO_PX = new int[] {
   private static final int[] STANDARD_LONG_EDGE_VIDEO_PX = new int[] {
       1920, 1600, 1440, 1280, 960, 854, 640, 540, 480};
       1920, 1600, 1440, 1280, 960, 854, 640, 540, 480};
 
 
+  // Generally there is zero or one pending output stream offset. We track more offsets to allow for
+  // pending output streams that have fewer frames than the codec latency.
+  private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10;
+
   private final VideoFrameReleaseTimeHelper frameReleaseTimeHelper;
   private final VideoFrameReleaseTimeHelper frameReleaseTimeHelper;
   private final EventDispatcher eventDispatcher;
   private final EventDispatcher eventDispatcher;
   private final long allowedJoiningTimeMs;
   private final long allowedJoiningTimeMs;
   private final int maxDroppedFramesToNotify;
   private final int maxDroppedFramesToNotify;
   private final boolean deviceNeedsAutoFrcWorkaround;
   private final boolean deviceNeedsAutoFrcWorkaround;
+  private final long[] pendingOutputStreamOffsetsUs;
 
 
   private Format[] streamFormats;
   private Format[] streamFormats;
   private CodecMaxValues codecMaxValues;
   private CodecMaxValues codecMaxValues;
@@ -95,6 +100,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private int tunnelingAudioSessionId;
   private int tunnelingAudioSessionId;
   /* package */ OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener;
   /* package */ OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener;
 
 
+  private long outputStreamOffsetUs;
+  private int pendingOutputStreamOffsetCount;
+
   /**
   /**
    * @param context A context.
    * @param context A context.
    * @param mediaCodecSelector A decoder selector.
    * @param mediaCodecSelector A decoder selector.
@@ -160,6 +168,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     frameReleaseTimeHelper = new VideoFrameReleaseTimeHelper(context);
     frameReleaseTimeHelper = new VideoFrameReleaseTimeHelper(context);
     eventDispatcher = new EventDispatcher(eventHandler, eventListener);
     eventDispatcher = new EventDispatcher(eventHandler, eventListener);
     deviceNeedsAutoFrcWorkaround = deviceNeedsAutoFrcWorkaround();
     deviceNeedsAutoFrcWorkaround = deviceNeedsAutoFrcWorkaround();
+    pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
+    outputStreamOffsetUs = C.TIME_UNSET;
     joiningDeadlineMs = C.TIME_UNSET;
     joiningDeadlineMs = C.TIME_UNSET;
     currentWidth = Format.NO_VALUE;
     currentWidth = Format.NO_VALUE;
     currentHeight = Format.NO_VALUE;
     currentHeight = Format.NO_VALUE;
@@ -219,9 +229,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
   }
 
 
   @Override
   @Override
-  protected void onStreamChanged(Format[] formats) throws ExoPlaybackException {
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
     streamFormats = formats;
     streamFormats = formats;
-    super.onStreamChanged(formats);
+    if (outputStreamOffsetUs == C.TIME_UNSET) {
+      outputStreamOffsetUs = offsetUs;
+    } else {
+      if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
+        Log.w(TAG, "Too many stream changes, so dropping offset: "
+            + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
+      } else {
+        pendingOutputStreamOffsetCount++;
+      }
+      pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs;
+    }
+    super.onStreamChanged(formats, offsetUs);
   }
   }
 
 
   @Override
   @Override
@@ -229,6 +250,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     super.onPositionReset(positionUs, joining);
     super.onPositionReset(positionUs, joining);
     clearRenderedFirstFrame();
     clearRenderedFirstFrame();
     consecutiveDroppedFrameCount = 0;
     consecutiveDroppedFrameCount = 0;
+    if (pendingOutputStreamOffsetCount != 0) {
+      outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1];
+      pendingOutputStreamOffsetCount = 0;
+    }
     if (joining) {
     if (joining) {
       setJoiningDeadlineMs();
       setJoiningDeadlineMs();
     } else {
     } else {
@@ -275,6 +300,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     currentHeight = Format.NO_VALUE;
     currentHeight = Format.NO_VALUE;
     currentPixelWidthHeightRatio = Format.NO_VALUE;
     currentPixelWidthHeightRatio = Format.NO_VALUE;
     pendingPixelWidthHeightRatio = Format.NO_VALUE;
     pendingPixelWidthHeightRatio = Format.NO_VALUE;
+    outputStreamOffsetUs = C.TIME_UNSET;
+    pendingOutputStreamOffsetCount = 0;
     clearReportedVideoSize();
     clearReportedVideoSize();
     clearRenderedFirstFrame();
     clearRenderedFirstFrame();
     frameReleaseTimeHelper.disable();
     frameReleaseTimeHelper.disable();
@@ -417,16 +444,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
   protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
       ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
       ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
       boolean shouldSkip) {
       boolean shouldSkip) {
+    while (pendingOutputStreamOffsetCount != 0
+        && bufferPresentationTimeUs >= pendingOutputStreamOffsetsUs[0]) {
+      outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
+      pendingOutputStreamOffsetCount--;
+      System.arraycopy(pendingOutputStreamOffsetsUs, 1, pendingOutputStreamOffsetsUs, 0,
+          pendingOutputStreamOffsetCount);
+    }
+    long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
     if (shouldSkip) {
     if (shouldSkip) {
-      skipOutputBuffer(codec, bufferIndex);
+      skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
       return true;
       return true;
     }
     }
 
 
     if (!renderedFirstFrame) {
     if (!renderedFirstFrame) {
       if (Util.SDK_INT >= 21) {
       if (Util.SDK_INT >= 21) {
-        renderOutputBufferV21(codec, bufferIndex, System.nanoTime());
+        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime());
       } else {
       } else {
-        renderOutputBuffer(codec, bufferIndex);
+        renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
       }
       }
       return true;
       return true;
     }
     }
@@ -450,14 +485,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
 
 
     if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) {
     if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) {
       // We're more than 30ms late rendering the frame.
       // We're more than 30ms late rendering the frame.
-      dropOutputBuffer(codec, bufferIndex);
+      dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
       return true;
       return true;
     }
     }
 
 
     if (Util.SDK_INT >= 21) {
     if (Util.SDK_INT >= 21) {
       // Let the underlying framework time the release.
       // Let the underlying framework time the release.
       if (earlyUs < 50000) {
       if (earlyUs < 50000) {
-        renderOutputBufferV21(codec, bufferIndex, adjustedReleaseTimeNs);
+        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs);
         return true;
         return true;
       }
       }
     } else {
     } else {
@@ -473,7 +508,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
             Thread.currentThread().interrupt();
             Thread.currentThread().interrupt();
           }
           }
         }
         }
-        renderOutputBuffer(codec, bufferIndex);
+        renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
         return true;
         return true;
       }
       }
     }
     }
@@ -495,16 +530,30 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     return earlyUs < -30000;
     return earlyUs < -30000;
   }
   }
 
 
-  private void skipOutputBuffer(MediaCodec codec, int bufferIndex) {
+  /**
+   * Skips the output buffer with the specified index.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param index The index of the output buffer to skip.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   */
+  protected void skipOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) {
     TraceUtil.beginSection("skipVideoBuffer");
     TraceUtil.beginSection("skipVideoBuffer");
-    codec.releaseOutputBuffer(bufferIndex, false);
+    codec.releaseOutputBuffer(index, false);
     TraceUtil.endSection();
     TraceUtil.endSection();
     decoderCounters.skippedOutputBufferCount++;
     decoderCounters.skippedOutputBufferCount++;
   }
   }
 
 
-  private void dropOutputBuffer(MediaCodec codec, int bufferIndex) {
+  /**
+   * Drops the output buffer with the specified index.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param index The index of the output buffer to drop.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   */
+  protected void dropOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) {
     TraceUtil.beginSection("dropVideoBuffer");
     TraceUtil.beginSection("dropVideoBuffer");
-    codec.releaseOutputBuffer(bufferIndex, false);
+    codec.releaseOutputBuffer(index, false);
     TraceUtil.endSection();
     TraceUtil.endSection();
     decoderCounters.droppedOutputBufferCount++;
     decoderCounters.droppedOutputBufferCount++;
     droppedFrames++;
     droppedFrames++;
@@ -516,21 +565,39 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     }
     }
   }
   }
 
 
-  private void renderOutputBuffer(MediaCodec codec, int bufferIndex) {
+  /**
+   * Renders the output buffer with the specified index. This method is only called if the platform
+   * API version of the device is less than 21.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param index The index of the output buffer to drop.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   */
+  protected void renderOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) {
     maybeNotifyVideoSizeChanged();
     maybeNotifyVideoSizeChanged();
     TraceUtil.beginSection("releaseOutputBuffer");
     TraceUtil.beginSection("releaseOutputBuffer");
-    codec.releaseOutputBuffer(bufferIndex, true);
+    codec.releaseOutputBuffer(index, true);
     TraceUtil.endSection();
     TraceUtil.endSection();
     decoderCounters.renderedOutputBufferCount++;
     decoderCounters.renderedOutputBufferCount++;
     consecutiveDroppedFrameCount = 0;
     consecutiveDroppedFrameCount = 0;
     maybeNotifyRenderedFirstFrame();
     maybeNotifyRenderedFirstFrame();
   }
   }
 
 
+  /**
+   * Renders the output buffer with the specified index. This method is only called if the platform
+   * API version of the device is 21 or later.
+   *
+   * @param codec The codec that owns the output buffer.
+   * @param index The index of the output buffer to drop.
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds.
+   */
   @TargetApi(21)
   @TargetApi(21)
-  private void renderOutputBufferV21(MediaCodec codec, int bufferIndex, long releaseTimeNs) {
+  protected void renderOutputBufferV21(MediaCodec codec, int index, long presentationTimeUs,
+      long releaseTimeNs) {
     maybeNotifyVideoSizeChanged();
     maybeNotifyVideoSizeChanged();
     TraceUtil.beginSection("releaseOutputBuffer");
     TraceUtil.beginSection("releaseOutputBuffer");
-    codec.releaseOutputBuffer(bufferIndex, releaseTimeNs);
+    codec.releaseOutputBuffer(index, releaseTimeNs);
     TraceUtil.endSection();
     TraceUtil.endSection();
     decoderCounters.renderedOutputBufferCount++;
     decoderCounters.renderedOutputBufferCount++;
     consecutiveDroppedFrameCount = 0;
     consecutiveDroppedFrameCount = 0;