Improve DASH/SS seek performance

This change enables feeding decoders from the closest sync frame
before a specified seek position, where-as previously we'd
always feed decoders from the start of the chunk. This avoids
decoding and discarding many audio samples during each seek. The
same benefit also applies to video chunks containing more than
one key-frame.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=190539547

RELEASENOTES.md

@@ -2,6 +2,8 @@
 
 ### dev-v2 (not yet released) ###
 
+* Optimize seeking in FMP4 by enabling seeking to the nearest sync sample within
+  a fragment. This benefits standalone FMP4 playbacks, DASH and SmoothStreaming.
 * Optimize seeking in FMP4.
 * Match codecs starting with "mp4a" to different Audio MimeTypes
   ([#3779](https://github.com/google/ExoPlayer/issues/3779)).

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/BaseMediaChunk.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.source.chunk;
 
+import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.upstream.DataSource;
 import com.google.android.exoplayer2.upstream.DataSpec;
@@ -24,6 +25,12 @@ import com.google.android.exoplayer2.upstream.DataSpec;
  */
 public abstract class BaseMediaChunk extends MediaChunk {
 
+  /**
+   * The media time from which output will begin, or {@link C#TIME_UNSET} if the whole chunk should
+   * be output.
+   */
+  public final long seekTimeUs;
+
   private BaseMediaChunkOutput output;
   private int[] firstSampleIndices;
 
@@ -35,6 +42,8 @@ public abstract class BaseMediaChunk extends MediaChunk {
    * @param trackSelectionData See {@link #trackSelectionData}.
    * @param startTimeUs The start time of the media contained by the chunk, in microseconds.
    * @param endTimeUs The end time of the media contained by the chunk, in microseconds.
+   * @param seekTimeUs The media time from which output will begin, or {@link C#TIME_UNSET} if the
+   *     whole chunk should be output.
    * @param chunkIndex The index of the chunk.
    */
   public BaseMediaChunk(
@@ -45,9 +54,11 @@ public abstract class BaseMediaChunk extends MediaChunk {
       Object trackSelectionData,
       long startTimeUs,
       long endTimeUs,
+      long seekTimeUs,
       long chunkIndex) {
     super(dataSource, dataSpec, trackFormat, trackSelectionReason, trackSelectionData, startTimeUs,
         endTimeUs, chunkIndex);
+    this.seekTimeUs = seekTimeUs;
   }
 
   /**

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/ChunkExtractorWrapper.java

@@ -96,18 +96,23 @@ public final class ChunkExtractorWrapper implements ExtractorOutput {
   }
 
   /**
-   * Initializes the wrapper to output to {@link TrackOutput}s provided by the specified
-   * {@link TrackOutputProvider}, and configures the extractor to receive data from a new chunk.
+   * Initializes the wrapper to output to {@link TrackOutput}s provided by the specified {@link
+   * TrackOutputProvider}, and configures the extractor to receive data from a new chunk.
    *
    * @param trackOutputProvider The provider of {@link TrackOutput}s that will receive sample data.
+   * @param seekTimeUs The seek position within the new chunk, or {@link C#TIME_UNSET} to output the
+   *     whole chunk.
    */
-  public void init(TrackOutputProvider trackOutputProvider) {
+  public void init(TrackOutputProvider trackOutputProvider, long seekTimeUs) {
     this.trackOutputProvider = trackOutputProvider;
     if (!extractorInitialized) {
       extractor.init(this);
+      if (seekTimeUs != C.TIME_UNSET) {
+        extractor.seek(/* position= */ 0, seekTimeUs);
+      }
       extractorInitialized = true;
     } else {
-      extractor.seek(0, 0);
+      extractor.seek(/* position= */ 0, seekTimeUs == C.TIME_UNSET ? 0 : seekTimeUs);
       for (int i = 0; i < bindingTrackOutputs.size(); i++) {
         bindingTrackOutputs.valueAt(i).bind(trackOutputProvider);
       }

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/ChunkSampleStream.java

@@ -242,7 +242,7 @@ public class ChunkSampleStream<T extends ChunkSource> implements SampleStream, S
       for (int i = 0; i < mediaChunks.size(); i++) {
         BaseMediaChunk mediaChunk = mediaChunks.get(i);
         long mediaChunkStartTimeUs = mediaChunk.startTimeUs;
-        if (mediaChunkStartTimeUs == positionUs) {
+        if (mediaChunkStartTimeUs == positionUs && mediaChunk.seekTimeUs == C.TIME_UNSET) {
           seekToMediaChunk = mediaChunk;
           break;
         } else if (mediaChunkStartTimeUs > positionUs) {

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/ContainerMediaChunk.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.source.chunk;
 
+import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.extractor.DefaultExtractorInput;
 import com.google.android.exoplayer2.extractor.Extractor;
@@ -46,6 +47,8 @@ public class ContainerMediaChunk extends BaseMediaChunk {
    * @param trackSelectionData See {@link #trackSelectionData}.
    * @param startTimeUs The start time of the media contained by the chunk, in microseconds.
    * @param endTimeUs The end time of the media contained by the chunk, in microseconds.
+   * @param seekTimeUs The media time from which output will begin, or {@link C#TIME_UNSET} if the
+   *     whole chunk should be output.
    * @param chunkIndex The index of the chunk.
    * @param chunkCount The number of chunks in the underlying media that are spanned by this
    *     instance. Normally equal to one, but may be larger if multiple chunks as defined by the
@@ -61,12 +64,21 @@ public class ContainerMediaChunk extends BaseMediaChunk {
       Object trackSelectionData,
       long startTimeUs,
       long endTimeUs,
+      long seekTimeUs,
       long chunkIndex,
       int chunkCount,
       long sampleOffsetUs,
       ChunkExtractorWrapper extractorWrapper) {
-    super(dataSource, dataSpec, trackFormat, trackSelectionReason, trackSelectionData, startTimeUs,
-        endTimeUs, chunkIndex);
+    super(
+        dataSource,
+        dataSpec,
+        trackFormat,
+        trackSelectionReason,
+        trackSelectionData,
+        startTimeUs,
+        endTimeUs,
+        seekTimeUs,
+        chunkIndex);
     this.chunkCount = chunkCount;
     this.sampleOffsetUs = sampleOffsetUs;
     this.extractorWrapper = extractorWrapper;
@@ -111,7 +123,8 @@ public class ContainerMediaChunk extends BaseMediaChunk {
         // Configure the output and set it as the target for the extractor wrapper.
         BaseMediaChunkOutput output = getOutput();
         output.setSampleOffsetUs(sampleOffsetUs);
-        extractorWrapper.init(output);
+        extractorWrapper.init(
+            output, seekTimeUs == C.TIME_UNSET ? 0 : (seekTimeUs - sampleOffsetUs));
       }
       // Load and decode the sample data.
       try {

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/InitializationChunk.java

@@ -78,7 +78,7 @@ public final class InitializationChunk extends Chunk {
       ExtractorInput input = new DefaultExtractorInput(dataSource,
           loadDataSpec.absoluteStreamPosition, dataSource.open(loadDataSpec));
       if (bytesLoaded == 0) {
-        extractorWrapper.init(null);
+        extractorWrapper.init(/* trackOutputProvider= */ null, C.TIME_UNSET);
       }
       // Load and decode the initialization data.
       try {

library/core/src/main/java/com/google/android/exoplayer2/source/chunk/SingleSampleMediaChunk.java

@@ -61,8 +61,16 @@ public final class SingleSampleMediaChunk extends BaseMediaChunk {
       long chunkIndex,
       int trackType,
       Format sampleFormat) {
-    super(dataSource, dataSpec, trackFormat, trackSelectionReason, trackSelectionData, startTimeUs,
-        endTimeUs, chunkIndex);
+    super(
+        dataSource,
+        dataSpec,
+        trackFormat,
+        trackSelectionReason,
+        trackSelectionData,
+        startTimeUs,
+        endTimeUs,
+        C.TIME_UNSET,
+        chunkIndex);
     this.trackType = trackType;
     this.sampleFormat = sampleFormat;
   }

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java

@@ -328,9 +328,18 @@ public class DefaultDashChunkSource implements DashChunkSource {
 
     int maxSegmentCount =
         (int) Math.min(maxSegmentsPerLoad, lastAvailableSegmentNum - segmentNum + 1);
-    out.chunk = newMediaChunk(representationHolder, dataSource, trackType,
-        trackSelection.getSelectedFormat(), trackSelection.getSelectionReason(),
-        trackSelection.getSelectionData(), segmentNum, maxSegmentCount);
+    long seekTimeUs = previous == null ? loadPositionUs : C.TIME_UNSET;
+    out.chunk =
+        newMediaChunk(
+            representationHolder,
+            dataSource,
+            trackType,
+            trackSelection.getSelectedFormat(),
+            trackSelection.getSelectionReason(),
+            trackSelection.getSelectionData(),
+            segmentNum,
+            maxSegmentCount,
+            seekTimeUs);
   }
 
   @Override
@@ -442,7 +451,8 @@ public class DefaultDashChunkSource implements DashChunkSource {
       int trackSelectionReason,
       Object trackSelectionData,
       long firstSegmentNum,
-      int maxSegmentCount) {
+      int maxSegmentCount,
+      long seekTimeUs) {
     Representation representation = representationHolder.representation;
     long startTimeUs = representationHolder.getSegmentStartTimeUs(firstSegmentNum);
     RangedUri segmentUri = representationHolder.getSegmentUrl(firstSegmentNum);
@@ -469,9 +479,19 @@ public class DefaultDashChunkSource implements DashChunkSource {
       DataSpec dataSpec = new DataSpec(segmentUri.resolveUri(baseUrl),
           segmentUri.start, segmentUri.length, representation.getCacheKey());
       long sampleOffsetUs = -representation.presentationTimeOffsetUs;
-      return new ContainerMediaChunk(dataSource, dataSpec, trackFormat, trackSelectionReason,
-          trackSelectionData, startTimeUs, endTimeUs, firstSegmentNum, segmentCount,
-          sampleOffsetUs, representationHolder.extractorWrapper);
+      return new ContainerMediaChunk(
+          dataSource,
+          dataSpec,
+          trackFormat,
+          trackSelectionReason,
+          trackSelectionData,
+          startTimeUs,
+          endTimeUs,
+          seekTimeUs,
+          firstSegmentNum,
+          segmentCount,
+          sampleOffsetUs,
+          representationHolder.extractorWrapper);
     }
   }
 

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/DefaultSsChunkSource.java

@@ -203,6 +203,7 @@ public class DefaultSsChunkSource implements SsChunkSource {
 
     long chunkStartTimeUs = streamElement.getStartTimeUs(chunkIndex);
     long chunkEndTimeUs = chunkStartTimeUs + streamElement.getChunkDurationUs(chunkIndex);
+    long chunkSeekTimeUs = previous == null ? loadPositionUs : C.TIME_UNSET;
     int currentAbsoluteChunkIndex = chunkIndex + currentManifestChunkOffset;
 
     int trackSelectionIndex = trackSelection.getSelectedIndex();
@@ -211,9 +212,19 @@ public class DefaultSsChunkSource implements SsChunkSource {
     int manifestTrackIndex = trackSelection.getIndexInTrackGroup(trackSelectionIndex);
     Uri uri = streamElement.buildRequestUri(manifestTrackIndex, chunkIndex);
 
-    out.chunk = newMediaChunk(trackSelection.getSelectedFormat(), dataSource, uri, null,
-        currentAbsoluteChunkIndex, chunkStartTimeUs, chunkEndTimeUs,
-        trackSelection.getSelectionReason(), trackSelection.getSelectionData(), extractorWrapper);
+    out.chunk =
+        newMediaChunk(
+            trackSelection.getSelectedFormat(),
+            dataSource,
+            uri,
+            null,
+            currentAbsoluteChunkIndex,
+            chunkStartTimeUs,
+            chunkEndTimeUs,
+            chunkSeekTimeUs,
+            trackSelection.getSelectionReason(),
+            trackSelection.getSelectionData(),
+            extractorWrapper);
   }
 
   @Override
@@ -229,15 +240,34 @@ public class DefaultSsChunkSource implements SsChunkSource {
 
   // Private methods.
 
-  private static MediaChunk newMediaChunk(Format format, DataSource dataSource, Uri uri,
-      String cacheKey, int chunkIndex, long chunkStartTimeUs, long chunkEndTimeUs,
-      int trackSelectionReason, Object trackSelectionData, ChunkExtractorWrapper extractorWrapper) {
+  private static MediaChunk newMediaChunk(
+      Format format,
+      DataSource dataSource,
+      Uri uri,
+      String cacheKey,
+      int chunkIndex,
+      long chunkStartTimeUs,
+      long chunkEndTimeUs,
+      long chunkSeekTimeUs,
+      int trackSelectionReason,
+      Object trackSelectionData,
+      ChunkExtractorWrapper extractorWrapper) {
     DataSpec dataSpec = new DataSpec(uri, 0, C.LENGTH_UNSET, cacheKey);
     // In SmoothStreaming each chunk contains sample timestamps relative to the start of the chunk.
     // To convert them the absolute timestamps, we need to set sampleOffsetUs to chunkStartTimeUs.
     long sampleOffsetUs = chunkStartTimeUs;
-    return new ContainerMediaChunk(dataSource, dataSpec, format, trackSelectionReason,
-        trackSelectionData, chunkStartTimeUs, chunkEndTimeUs, chunkIndex, 1, sampleOffsetUs,
+    return new ContainerMediaChunk(
+        dataSource,
+        dataSpec,
+        format,
+        trackSelectionReason,
+        trackSelectionData,
+        chunkStartTimeUs,
+        chunkEndTimeUs,
+        chunkSeekTimeUs,
+        chunkIndex,
+        /* chunkCount= */ 1,
+        sampleOffsetUs,
         extractorWrapper);
   }