Add option to MergingMediaSource to adjust for period time diffs

Without this option it's impossible to merge periods covering
different timestamps (at least not without playback issues).

Issue:issue:#6103
PiperOrigin-RevId: 299817540

RELEASENOTES.md

@@ -47,6 +47,9 @@
         ([#2863](https://github.com/google/ExoPlayer/issues/2863)).
         ([#2863](https://github.com/google/ExoPlayer/issues/2863)).
     *   Add optional automatic `WifiLock` handling to `SimpleExoPlayer`
     *   Add optional automatic `WifiLock` handling to `SimpleExoPlayer`
         ([#6914](https://github.com/google/ExoPlayer/issues/6914)).
         ([#6914](https://github.com/google/ExoPlayer/issues/6914)).
+    *   Add option to `MergingMediaSource` to adjust the time offsets between
+        the merged sources
+        ([#6103](https://github.com/google/ExoPlayer/issues/6103)).
 *   Text:
 *   Text:
     *   Parse `<ruby>` and `<rt>` tags in WebVTT subtitles (rendering is coming
     *   Parse `<ruby>` and `<rt>` tags in WebVTT subtitles (rendering is coming
         later).
         later).

library/core/src/main/java/com/google/android/exoplayer2/source/MergingMediaSource.java

@@ -66,34 +66,59 @@ public final class MergingMediaSource extends CompositeMediaSource<Integer> {
 
 
   private static final int PERIOD_COUNT_UNSET = -1;
   private static final int PERIOD_COUNT_UNSET = -1;
 
 
+  private final boolean adjustPeriodTimeOffsets;
   private final MediaSource[] mediaSources;
   private final MediaSource[] mediaSources;
   private final Timeline[] timelines;
   private final Timeline[] timelines;
   private final ArrayList<MediaSource> pendingTimelineSources;
   private final ArrayList<MediaSource> pendingTimelineSources;
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
 
 
   private int periodCount;
   private int periodCount;
+  private long[][] periodTimeOffsetsUs;
   @Nullable private IllegalMergeException mergeError;
   @Nullable private IllegalMergeException mergeError;
 
 
   /**
   /**
+   * Creates a merging media source.
+   *
+   * <p>Offsets between the timestamps in the media sources will not be adjusted.
+   *
    * @param mediaSources The {@link MediaSource}s to merge.
    * @param mediaSources The {@link MediaSource}s to merge.
    */
    */
   public MergingMediaSource(MediaSource... mediaSources) {
   public MergingMediaSource(MediaSource... mediaSources) {
-    this(new DefaultCompositeSequenceableLoaderFactory(), mediaSources);
+    this(/* adjustPeriodTimeOffsets= */ false, mediaSources);
+  }
+
+  /**
+   * Creates a merging media source.
+   *
+   * @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all
+   *     start at the same time.
+   * @param mediaSources The {@link MediaSource}s to merge.
+   */
+  public MergingMediaSource(boolean adjustPeriodTimeOffsets, MediaSource... mediaSources) {
+    this(adjustPeriodTimeOffsets, new DefaultCompositeSequenceableLoaderFactory(), mediaSources);
   }
   }
 
 
   /**
   /**
-   * @param compositeSequenceableLoaderFactory A factory to create composite
-   *     {@link SequenceableLoader}s for when this media source loads data from multiple streams
-   *     (video, audio etc...).
+   * Creates a merging media source.
+   *
+   * @param adjustPeriodTimeOffsets Whether to adjust timestamps of the merged media sources to all
+   *     start at the same time.
+   * @param compositeSequenceableLoaderFactory A factory to create composite {@link
+   *     SequenceableLoader}s for when this media source loads data from multiple streams (video,
+   *     audio etc...).
    * @param mediaSources The {@link MediaSource}s to merge.
    * @param mediaSources The {@link MediaSource}s to merge.
    */
    */
-  public MergingMediaSource(CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
+  public MergingMediaSource(
+      boolean adjustPeriodTimeOffsets,
+      CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
       MediaSource... mediaSources) {
       MediaSource... mediaSources) {
+    this.adjustPeriodTimeOffsets = adjustPeriodTimeOffsets;
     this.mediaSources = mediaSources;
     this.mediaSources = mediaSources;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
     pendingTimelineSources = new ArrayList<>(Arrays.asList(mediaSources));
     pendingTimelineSources = new ArrayList<>(Arrays.asList(mediaSources));
     periodCount = PERIOD_COUNT_UNSET;
     periodCount = PERIOD_COUNT_UNSET;
     timelines = new Timeline[mediaSources.length];
     timelines = new Timeline[mediaSources.length];
+    periodTimeOffsetsUs = new long[0][];
   }
   }
 
 
   @Override
   @Override
@@ -125,16 +150,19 @@ public final class MergingMediaSource extends CompositeMediaSource<Integer> {
     for (int i = 0; i < periods.length; i++) {
     for (int i = 0; i < periods.length; i++) {
       MediaPeriodId childMediaPeriodId =
       MediaPeriodId childMediaPeriodId =
           id.copyWithPeriodUid(timelines[i].getUidOfPeriod(periodIndex));
           id.copyWithPeriodUid(timelines[i].getUidOfPeriod(periodIndex));
-      periods[i] = mediaSources[i].createPeriod(childMediaPeriodId, allocator, startPositionUs);
+      periods[i] =
+          mediaSources[i].createPeriod(
+              childMediaPeriodId, allocator, startPositionUs - periodTimeOffsetsUs[periodIndex][i]);
     }
     }
-    return new MergingMediaPeriod(compositeSequenceableLoaderFactory, periods);
+    return new MergingMediaPeriod(
+        compositeSequenceableLoaderFactory, periodTimeOffsetsUs[periodIndex], periods);
   }
   }
 
 
   @Override
   @Override
   public void releasePeriod(MediaPeriod mediaPeriod) {
   public void releasePeriod(MediaPeriod mediaPeriod) {
     MergingMediaPeriod mergingPeriod = (MergingMediaPeriod) mediaPeriod;
     MergingMediaPeriod mergingPeriod = (MergingMediaPeriod) mediaPeriod;
     for (int i = 0; i < mediaSources.length; i++) {
     for (int i = 0; i < mediaSources.length; i++) {
-      mediaSources[i].releasePeriod(mergingPeriod.periods[i]);
+      mediaSources[i].releasePeriod(mergingPeriod.getChildPeriod(i));
     }
     }
   }
   }
 
 
@@ -151,15 +179,24 @@ public final class MergingMediaSource extends CompositeMediaSource<Integer> {
   @Override
   @Override
   protected void onChildSourceInfoRefreshed(
   protected void onChildSourceInfoRefreshed(
       Integer id, MediaSource mediaSource, Timeline timeline) {
       Integer id, MediaSource mediaSource, Timeline timeline) {
-    if (mergeError == null) {
-      mergeError = checkTimelineMerges(timeline);
-    }
     if (mergeError != null) {
     if (mergeError != null) {
       return;
       return;
     }
     }
+    if (periodCount == PERIOD_COUNT_UNSET) {
+      periodCount = timeline.getPeriodCount();
+    } else if (timeline.getPeriodCount() != periodCount) {
+      mergeError = new IllegalMergeException(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH);
+      return;
+    }
+    if (periodTimeOffsetsUs.length == 0) {
+      periodTimeOffsetsUs = new long[periodCount][timelines.length];
+    }
     pendingTimelineSources.remove(mediaSource);
     pendingTimelineSources.remove(mediaSource);
     timelines[id] = timeline;
     timelines[id] = timeline;
     if (pendingTimelineSources.isEmpty()) {
     if (pendingTimelineSources.isEmpty()) {
+      if (adjustPeriodTimeOffsets) {
+        computePeriodTimeOffsets();
+      }
       refreshSourceInfo(timelines[0]);
       refreshSourceInfo(timelines[0]);
     }
     }
   }
   }
@@ -171,14 +208,17 @@ public final class MergingMediaSource extends CompositeMediaSource<Integer> {
     return id == 0 ? mediaPeriodId : null;
     return id == 0 ? mediaPeriodId : null;
   }
   }
 
 
-  @Nullable
-  private IllegalMergeException checkTimelineMerges(Timeline timeline) {
-    if (periodCount == PERIOD_COUNT_UNSET) {
-      periodCount = timeline.getPeriodCount();
-    } else if (timeline.getPeriodCount() != periodCount) {
-      return new IllegalMergeException(IllegalMergeException.REASON_PERIOD_COUNT_MISMATCH);
+  private void computePeriodTimeOffsets() {
+    Timeline.Period period = new Timeline.Period();
+    for (int periodIndex = 0; periodIndex < periodCount; periodIndex++) {
+      long primaryWindowOffsetUs =
+          -timelines[0].getPeriod(periodIndex, period).getPositionInWindowUs();
+      for (int timelineIndex = 1; timelineIndex < timelines.length; timelineIndex++) {
+        long secondaryWindowOffsetUs =
+            -timelines[timelineIndex].getPeriod(periodIndex, period).getPositionInWindowUs();
+        periodTimeOffsetsUs[periodIndex][timelineIndex] =
+            primaryWindowOffsetUs - secondaryWindowOffsetUs;
+      }
     }
     }
-    return null;
   }
   }
-
 }
 }

library/core/src/test/java/com/google/android/exoplayer2/source/MergingMediaPeriodTest.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.source;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.FormatHolder;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.source.MediaSourceEventListener.EventDispatcher;
+import com.google.android.exoplayer2.testutil.FakeMediaPeriod;
+import com.google.android.exoplayer2.trackselection.FixedTrackSelection;
+import com.google.android.exoplayer2.trackselection.TrackSelection;
+import java.util.concurrent.CountDownLatch;
+import org.checkerframework.checker.nullness.compatqual.NullableType;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.LooperMode;
+
+/** Unit test for {@link MergingMediaPeriod}. */
+@RunWith(AndroidJUnit4.class)
+@LooperMode(LooperMode.Mode.PAUSED)
+public final class MergingMediaPeriodTest {
+
+  private static final Format childFormat11 = new Format.Builder().setId("1_1").build();
+  private static final Format childFormat12 = new Format.Builder().setId("1_2").build();
+  private static final Format childFormat21 = new Format.Builder().setId("2_1").build();
+  private static final Format childFormat22 = new Format.Builder().setId("2_2").build();
+
+  @Test
+  public void getTrackGroups_returnsAllChildTrackGroups() throws Exception {
+    MergingMediaPeriod mergingMediaPeriod =
+        prepareMergingPeriod(
+            new MergingPeriodDefinition(/* timeOffsetUs= */ 0, childFormat11, childFormat12),
+            new MergingPeriodDefinition(/* timeOffsetUs= */ 0, childFormat21, childFormat22));
+
+    assertThat(mergingMediaPeriod.getTrackGroups().length).isEqualTo(4);
+    assertThat(mergingMediaPeriod.getTrackGroups().get(0).getFormat(0)).isEqualTo(childFormat11);
+    assertThat(mergingMediaPeriod.getTrackGroups().get(1).getFormat(0)).isEqualTo(childFormat12);
+    assertThat(mergingMediaPeriod.getTrackGroups().get(2).getFormat(0)).isEqualTo(childFormat21);
+    assertThat(mergingMediaPeriod.getTrackGroups().get(3).getFormat(0)).isEqualTo(childFormat22);
+  }
+
+  @Test
+  public void selectTracks_createsSampleStreamsFromChildPeriods() throws Exception {
+    MergingMediaPeriod mergingMediaPeriod =
+        prepareMergingPeriod(
+            new MergingPeriodDefinition(/* timeOffsetUs= */ 0, childFormat11, childFormat12),
+            new MergingPeriodDefinition(/* timeOffsetUs= */ 0, childFormat21, childFormat22));
+
+    TrackSelection selectionForChild1 =
+        new FixedTrackSelection(mergingMediaPeriod.getTrackGroups().get(1), /* track= */ 0);
+    TrackSelection selectionForChild2 =
+        new FixedTrackSelection(mergingMediaPeriod.getTrackGroups().get(2), /* track= */ 0);
+    SampleStream[] streams = new SampleStream[4];
+    mergingMediaPeriod.selectTracks(
+        /* selections= */ new TrackSelection[] {null, selectionForChild1, selectionForChild2, null},
+        /* mayRetainStreamFlags= */ new boolean[] {false, false, false, false},
+        streams,
+        /* streamResetFlags= */ new boolean[] {false, false, false, false},
+        /* positionUs= */ 0);
+
+    assertThat(streams[0]).isNull();
+    assertThat(streams[3]).isNull();
+
+    FormatHolder formatHolder = new FormatHolder();
+    DecoderInputBuffer inputBuffer =
+        new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL);
+    assertThat(streams[1].readData(formatHolder, inputBuffer, /* formatRequired= */ true))
+        .isEqualTo(C.RESULT_FORMAT_READ);
+    assertThat(formatHolder.format).isEqualTo(childFormat12);
+
+    assertThat(streams[2].readData(formatHolder, inputBuffer, /* formatRequired= */ true))
+        .isEqualTo(C.RESULT_FORMAT_READ);
+    assertThat(formatHolder.format).isEqualTo(childFormat21);
+  }
+
+  @Test
+  public void
+      selectTracks_withPeriodOffsets_selectTracksWithOffset_andCreatesSampleStreamsCorrectingOffset()
+          throws Exception {
+    MergingMediaPeriod mergingMediaPeriod =
+        prepareMergingPeriod(
+            new MergingPeriodDefinition(/* timeOffsetUs= */ 0, childFormat11, childFormat12),
+            new MergingPeriodDefinition(/* timeOffsetUs= */ -3000, childFormat21, childFormat22));
+
+    TrackSelection selectionForChild1 =
+        new FixedTrackSelection(mergingMediaPeriod.getTrackGroups().get(0), /* track= */ 0);
+    TrackSelection selectionForChild2 =
+        new FixedTrackSelection(mergingMediaPeriod.getTrackGroups().get(2), /* track= */ 0);
+    SampleStream[] streams = new SampleStream[2];
+    mergingMediaPeriod.selectTracks(
+        /* selections= */ new TrackSelection[] {selectionForChild1, selectionForChild2},
+        /* mayRetainStreamFlags= */ new boolean[] {false, false},
+        streams,
+        /* streamResetFlags= */ new boolean[] {false, false},
+        /* positionUs= */ 0);
+    FormatHolder formatHolder = new FormatHolder();
+    DecoderInputBuffer inputBuffer =
+        new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL);
+    streams[0].readData(formatHolder, inputBuffer, /* formatRequired= */ true);
+    streams[1].readData(formatHolder, inputBuffer, /* formatRequired= */ true);
+
+    FakeMediaPeriodWithSelectTracksPosition childMediaPeriod1 =
+        (FakeMediaPeriodWithSelectTracksPosition) mergingMediaPeriod.getChildPeriod(0);
+    assertThat(childMediaPeriod1.selectTracksPositionUs).isEqualTo(0);
+    assertThat(streams[0].readData(formatHolder, inputBuffer, /* formatRequired= */ false))
+        .isEqualTo(C.RESULT_BUFFER_READ);
+    assertThat(inputBuffer.timeUs).isEqualTo(0L);
+
+    FakeMediaPeriodWithSelectTracksPosition childMediaPeriod2 =
+        (FakeMediaPeriodWithSelectTracksPosition) mergingMediaPeriod.getChildPeriod(1);
+    assertThat(childMediaPeriod2.selectTracksPositionUs).isEqualTo(3000L);
+    assertThat(streams[1].readData(formatHolder, inputBuffer, /* formatRequired= */ false))
+        .isEqualTo(C.RESULT_BUFFER_READ);
+    assertThat(inputBuffer.timeUs).isEqualTo(0L);
+  }
+
+  private MergingMediaPeriod prepareMergingPeriod(MergingPeriodDefinition... definitions)
+      throws Exception {
+    MediaPeriod[] mediaPeriods = new MediaPeriod[definitions.length];
+    long[] timeOffsetsUs = new long[definitions.length];
+    for (int i = 0; i < definitions.length; i++) {
+      timeOffsetsUs[i] = definitions[i].timeOffsetUs;
+      TrackGroup[] trackGroups = new TrackGroup[definitions[i].formats.length];
+      for (int j = 0; j < definitions[i].formats.length; j++) {
+        trackGroups[j] = new TrackGroup(definitions[i].formats[j]);
+      }
+      mediaPeriods[i] =
+          new FakeMediaPeriodWithSelectTracksPosition(
+              new TrackGroupArray(trackGroups), new EventDispatcher());
+    }
+    MergingMediaPeriod mergingMediaPeriod =
+        new MergingMediaPeriod(
+            new DefaultCompositeSequenceableLoaderFactory(), timeOffsetsUs, mediaPeriods);
+
+    CountDownLatch prepareCountDown = new CountDownLatch(1);
+    mergingMediaPeriod.prepare(
+        new MediaPeriod.Callback() {
+          @Override
+          public void onPrepared(MediaPeriod mediaPeriod) {
+            prepareCountDown.countDown();
+          }
+
+          @Override
+          public void onContinueLoadingRequested(MediaPeriod source) {
+            mergingMediaPeriod.continueLoading(/* positionUs= */ 0);
+          }
+        },
+        /* positionUs= */ 0);
+    prepareCountDown.await();
+
+    return mergingMediaPeriod;
+  }
+
+  private static final class FakeMediaPeriodWithSelectTracksPosition extends FakeMediaPeriod {
+
+    public long selectTracksPositionUs;
+
+    public FakeMediaPeriodWithSelectTracksPosition(
+        TrackGroupArray trackGroupArray, EventDispatcher eventDispatcher) {
+      super(trackGroupArray, eventDispatcher);
+      selectTracksPositionUs = C.TIME_UNSET;
+    }
+
+    @Override
+    public long selectTracks(
+        @NullableType TrackSelection[] selections,
+        boolean[] mayRetainStreamFlags,
+        @NullableType SampleStream[] streams,
+        boolean[] streamResetFlags,
+        long positionUs) {
+      selectTracksPositionUs = positionUs;
+      return super.selectTracks(
+          selections, mayRetainStreamFlags, streams, streamResetFlags, positionUs);
+    }
+  }
+
+  private static final class MergingPeriodDefinition {
+
+    public long timeOffsetUs;
+    public Format[] formats;
+
+    public MergingPeriodDefinition(long timeOffsetUs, Format... formats) {
+      this.timeOffsetUs = timeOffsetUs;
+      this.formats = formats;
+    }
+  }
+}