Support multiple CC channels in DASH

Issue: #5656 PiperOrigin-RevId: 241235377

Support multiple CC channels in DASH
Issue: #5656 PiperOrigin-RevId: 241235377
592b5eaf · olly · Oliver Woodman · 5db33dee · 592b5eaf · 592b5eaf
Commit 592b5eaf authored Mar 31, 2019 by olly Committed by Oliver Woodman Apr 01, 2019
Showing with 148 additions and 78 deletions
RELEASENOTES.md
library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java
library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java
library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -5,7 +5,10 @@
 * Update to Mockito 2
 * Add new `ExoPlaybackException` types for remote exceptions and out-of-memory
  errors.
-* DASH: Parse role and accessibility descriptors into `Format.roleFlags`.
+* DASH:
+  * Parse role and accessibility descriptors into `Format.roleFlags`.
+  * Support multiple CEA-608 channels muxed into FMP4 representations
+    ([#5656](https://github.com/google/ExoPlayer/issues/5656)).
 * HLS:
  * Work around lack of LA_URL attribute in PlayReady key request init data.
  * Prevent unnecessary reloads of initialization segments.

--- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java
+++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java
@@ -17,12 +17,14 @@ package com.google.android.exoplayer2.source.dash;
 import android.os.SystemClock;
 import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.source.chunk.ChunkSource;
 import com.google.android.exoplayer2.source.dash.PlayerEmsgHandler.PlayerTrackEmsgHandler;
 import com.google.android.exoplayer2.source.dash.manifest.DashManifest;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.upstream.LoaderErrorThrower;
 import com.google.android.exoplayer2.upstream.TransferListener;
+import java.util.List;
 /**
 * An {@link ChunkSource} for DASH streams.
@@ -41,10 +43,8 @@ public interface DashChunkSource extends ChunkSource {
     * @param elapsedRealtimeOffsetMs If known, an estimate of the instantaneous difference between
     *     server-side unix time and {@link SystemClock#elapsedRealtime()} in milliseconds,
     *     specified as the server's unix time minus the local elapsed time. If unknown, set to 0.
-     * @param enableEventMessageTrack Whether the chunks generated by the source may output an event
+     * @param enableEventMessageTrack Whether to output an event message track.
-     *     message track.
+     * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output.
-     * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608
-     *     track.
     * @param transferListener The transfer listener which should be informed of any data transfers.
     *     May be null if no listener is available.
     * @return The created {@link DashChunkSource}.
@@ -58,7 +58,7 @@ public interface DashChunkSource extends ChunkSource {
        int type,
        long elapsedRealtimeOffsetMs,
        boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
        @Nullable PlayerTrackEmsgHandler playerEmsgHandler,
        @Nullable TransferListener transferListener);
  }

--- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java
+++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java
@@ -56,6 +56,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 /** A DASH {@link MediaPeriod}. */
 /* package */ final class DashMediaPeriod
@@ -63,6 +65,8 @@ import java.util.List;
        SequenceableLoader.Callback<ChunkSampleStream<DashChunkSource>>,
        ChunkSampleStream.ReleaseCallback<DashChunkSource> {
+  private static final Pattern CEA608_SERVICE_DESCRIPTOR_REGEX = Pattern.compile("CC([1-4])=(.+)");
  /* package */ final int id;
  private final DashChunkSource.Factory chunkSourceFactory;
  private final @Nullable TransferListener transferListener;
@@ -457,18 +461,28 @@ import java.util.List;
    int primaryGroupCount = groupedAdaptationSetIndices.length;
    boolean[] primaryGroupHasEventMessageTrackFlags = new boolean[primaryGroupCount];
-    boolean[] primaryGroupHasCea608TrackFlags = new boolean[primaryGroupCount];
+    Format[][] primaryGroupCea608TrackFormats = new Format[primaryGroupCount][];
-    int totalEmbeddedTrackGroupCount = identifyEmbeddedTracks(primaryGroupCount, adaptationSets,
+    int totalEmbeddedTrackGroupCount =
-        groupedAdaptationSetIndices, primaryGroupHasEventMessageTrackFlags,
+        identifyEmbeddedTracks(
-        primaryGroupHasCea608TrackFlags);
+            primaryGroupCount,
+            adaptationSets,
+            groupedAdaptationSetIndices,
+            primaryGroupHasEventMessageTrackFlags,
+            primaryGroupCea608TrackFormats);
    int totalGroupCount = primaryGroupCount + totalEmbeddedTrackGroupCount + eventStreams.size();
    TrackGroup[] trackGroups = new TrackGroup[totalGroupCount];
    TrackGroupInfo[] trackGroupInfos = new TrackGroupInfo[totalGroupCount];
-    int trackGroupCount = buildPrimaryAndEmbeddedTrackGroupInfos(adaptationSets,
+    int trackGroupCount =
-        groupedAdaptationSetIndices, primaryGroupCount, primaryGroupHasEventMessageTrackFlags,
+        buildPrimaryAndEmbeddedTrackGroupInfos(
-        primaryGroupHasCea608TrackFlags, trackGroups, trackGroupInfos);
+            adaptationSets,
+            groupedAdaptationSetIndices,
+            primaryGroupCount,
+            primaryGroupHasEventMessageTrackFlags,
+            primaryGroupCea608TrackFormats,
+            trackGroups,
+            trackGroupInfos);
    buildManifestEventTrackGroupInfos(eventStreams, trackGroups, trackGroupInfos, trackGroupCount);
@@ -524,39 +538,46 @@ import java.util.List;
  /**
   * Iterates through list of primary track groups and identifies embedded tracks.
-   * <p>
+   *
   * @param primaryGroupCount The number of primary track groups.
   * @param adaptationSets The list of {@link AdaptationSet} of the current DASH period.
-   * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to
+   * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to the
-   *     the same primary group, grouped in primary track groups order.
+   *     same primary group, grouped in primary track groups order.
-   * @param primaryGroupHasEventMessageTrackFlags An output array containing boolean flag, each
+   * @param primaryGroupHasEventMessageTrackFlags An output array to be filled with flags indicating
-   *     indicates whether the corresponding primary track group contains an embedded event message
+   *     whether each of the primary track groups contains an embedded event message track.
-   *     track.
+   * @param primaryGroupCea608TrackFormats An output array to be filled with track formats for
-   * @param primaryGroupHasCea608TrackFlags An output array containing boolean flag, each
+   *     CEA-608 tracks embedded in each of the primary track groups.
-   *     indicates whether the corresponding primary track group contains an embedded Cea608 track.
+   * @return Total number of embedded track groups.
-   * @return Total number of embedded tracks.
   */
-  private static int identifyEmbeddedTracks(int primaryGroupCount,
+  private static int identifyEmbeddedTracks(
-      List<AdaptationSet> adaptationSets, int[][] groupedAdaptationSetIndices,
+      int primaryGroupCount,
-      boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags) {
+      List<AdaptationSet> adaptationSets,
-    int numEmbeddedTrack = 0;
+      int[][] groupedAdaptationSetIndices,
+      boolean[] primaryGroupHasEventMessageTrackFlags,
+      Format[][] primaryGroupCea608TrackFormats) {
+    int numEmbeddedTrackGroups = 0;
    for (int i = 0; i < primaryGroupCount; i++) {
      if (hasEventMessageTrack(adaptationSets, groupedAdaptationSetIndices[i])) {
        primaryGroupHasEventMessageTrackFlags[i] = true;
-        numEmbeddedTrack++;
+        numEmbeddedTrackGroups++;
      }
-      if (hasCea608Track(adaptationSets, groupedAdaptationSetIndices[i])) {
+      primaryGroupCea608TrackFormats[i] =
-        primaryGroupHasCea608TrackFlags[i] = true;
+          getCea608TrackFormats(adaptationSets, groupedAdaptationSetIndices[i]);
-        numEmbeddedTrack++;
+      if (primaryGroupCea608TrackFormats[i].length != 0) {
+        numEmbeddedTrackGroups++;
      }
    }
-    return numEmbeddedTrack;
+    return numEmbeddedTrackGroups;
  }
-  private static int buildPrimaryAndEmbeddedTrackGroupInfos(List<AdaptationSet> adaptationSets,
+  private static int buildPrimaryAndEmbeddedTrackGroupInfos(
-      int[][] groupedAdaptationSetIndices, int primaryGroupCount,
+      List<AdaptationSet> adaptationSets,
-      boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags,
+      int[][] groupedAdaptationSetIndices,
-      TrackGroup[] trackGroups, TrackGroupInfo[] trackGroupInfos) {
+      int primaryGroupCount,
+      boolean[] primaryGroupHasEventMessageTrackFlags,
+      Format[][] primaryGroupCea608TrackFormats,
+      TrackGroup[] trackGroups,
+      TrackGroupInfo[] trackGroupInfos) {
    int trackGroupCount = 0;
    for (int i = 0; i < primaryGroupCount; i++) {
      int[] adaptationSetIndices = groupedAdaptationSetIndices[i];
@@ -574,7 +595,7 @@ import java.util.List;
      int eventMessageTrackGroupIndex =
          primaryGroupHasEventMessageTrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET;
      int cea608TrackGroupIndex =
-          primaryGroupHasCea608TrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET;
+          primaryGroupCea608TrackFormats[i].length != 0 ? trackGroupCount++ : C.INDEX_UNSET;
      trackGroups[primaryTrackGroupIndex] = new TrackGroup(formats);
      trackGroupInfos[primaryTrackGroupIndex] =
@@ -592,9 +613,7 @@ import java.util.List;
            TrackGroupInfo.embeddedEmsgTrack(adaptationSetIndices, primaryTrackGroupIndex);
      }
      if (cea608TrackGroupIndex != C.INDEX_UNSET) {
-        Format format = Format.createTextSampleFormat(firstAdaptationSet.id + ":cea608",
+        trackGroups[cea608TrackGroupIndex] = new TrackGroup(primaryGroupCea608TrackFormats[i]);
-            MimeTypes.APPLICATION_CEA608, 0, null);
-        trackGroups[cea608TrackGroupIndex] = new TrackGroup(format);
        trackGroupInfos[cea608TrackGroupIndex] =
            TrackGroupInfo.embeddedCea608Track(adaptationSetIndices, primaryTrackGroupIndex);
      }
@@ -616,25 +635,39 @@ import java.util.List;
  private ChunkSampleStream<DashChunkSource> buildSampleStream(TrackGroupInfo trackGroupInfo,
      TrackSelection selection, long positionUs) {
    int embeddedTrackCount = 0;
-    int[] embeddedTrackTypes = new int[2];
-    Format[] embeddedTrackFormats = new Format[2];
    boolean enableEventMessageTrack =
        trackGroupInfo.embeddedEventMessageTrackGroupIndex != C.INDEX_UNSET;
+    TrackGroup embeddedEventMessageTrackGroup = null;
    if (enableEventMessageTrack) {
-      embeddedTrackFormats[embeddedTrackCount] =
+      embeddedEventMessageTrackGroup =
-          trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex).getFormat(0);
+          trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex);
-      embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_METADATA;
+      embeddedTrackCount++;
    }
-    boolean enableCea608Track = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET;
+    boolean enableCea608Tracks = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET;
-    if (enableCea608Track) {
+    TrackGroup embeddedCea608TrackGroup = null;
-      embeddedTrackFormats[embeddedTrackCount] =
+    if (enableCea608Tracks) {
-          trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex).getFormat(0);
+      embeddedCea608TrackGroup = trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex);
-      embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_TEXT;
+      embeddedTrackCount += embeddedCea608TrackGroup.length;
    }
-    if (embeddedTrackCount < embeddedTrackTypes.length) {
-      embeddedTrackFormats = Arrays.copyOf(embeddedTrackFormats, embeddedTrackCount);
+    Format[] embeddedTrackFormats = new Format[embeddedTrackCount];
-      embeddedTrackTypes = Arrays.copyOf(embeddedTrackTypes, embeddedTrackCount);
+    int[] embeddedTrackTypes = new int[embeddedTrackCount];
+    embeddedTrackCount = 0;
+    if (enableEventMessageTrack) {
+      embeddedTrackFormats[embeddedTrackCount] = embeddedEventMessageTrackGroup.getFormat(0);
+      embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_METADATA;
+      embeddedTrackCount++;
+    }
+    List<Format> embeddedCea608TrackFormats = new ArrayList<>();
+    if (enableCea608Tracks) {
+      for (int i = 0; i < embeddedCea608TrackGroup.length; i++) {
+        embeddedTrackFormats[embeddedTrackCount] = embeddedCea608TrackGroup.getFormat(i);
+        embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_TEXT;
+        embeddedCea608TrackFormats.add(embeddedTrackFormats[embeddedTrackCount]);
+        embeddedTrackCount++;
+      }
    }
    PlayerTrackEmsgHandler trackPlayerEmsgHandler =
        manifest.dynamic && enableEventMessageTrack
            ? playerEmsgHandler.newPlayerTrackEmsgHandler()
@@ -649,7 +682,7 @@ import java.util.List;
            trackGroupInfo.trackType,
            elapsedRealtimeOffsetMs,
            enableEventMessageTrack,
-            enableCea608Track,
+            embeddedCea608TrackFormats,
            trackPlayerEmsgHandler,
            transferListener);
    ChunkSampleStream<DashChunkSource> stream =
@@ -694,18 +727,60 @@ import java.util.List;
    return false;
  }
-  private static boolean hasCea608Track(List<AdaptationSet> adaptationSets,
+  private static Format[] getCea608TrackFormats(
-      int[] adaptationSetIndices) {
+      List<AdaptationSet> adaptationSets, int[] adaptationSetIndices) {
    for (int i : adaptationSetIndices) {
+      AdaptationSet adaptationSet = adaptationSets.get(i);
      List<Descriptor> descriptors = adaptationSets.get(i).accessibilityDescriptors;
      for (int j = 0; j < descriptors.size(); j++) {
        Descriptor descriptor = descriptors.get(j);
        if ("urn:scte:dash:cc:cea-608:2015".equals(descriptor.schemeIdUri)) {
-          return true;
+          String value = descriptor.value;
+          if (value == null) {
+            // There are embedded CEA-608 tracks, but service information is not declared.
+            return new Format[] {buildCea608TrackFormat(adaptationSet.id)};
+          }
+          String[] services = Util.split(value, ";");
+          Format[] formats = new Format[services.length];
+          for (int k = 0; k < services.length; k++) {
+            Matcher matcher = CEA608_SERVICE_DESCRIPTOR_REGEX.matcher(services[k]);
+            if (!matcher.matches()) {
+              // If we can't parse service information for all services, assume a single track.
+              return new Format[] {buildCea608TrackFormat(adaptationSet.id)};
+            }
+            formats[k] =
+                buildCea608TrackFormat(
+                    adaptationSet.id,
+                    /* language= */ matcher.group(2),
+                    /* accessibilityChannel= */ Integer.parseInt(matcher.group(1)));
+          }
+          return formats;
        }
      }
    }
-    return false;
+    return new Format[0];
+  }
+  private static Format buildCea608TrackFormat(int adaptationSetId) {
+    return buildCea608TrackFormat(
+        adaptationSetId, /* language= */ null, /* accessibilityChannel= */ Format.NO_VALUE);
+  }
+  private static Format buildCea608TrackFormat(
+      int adaptationSetId, String language, int accessibilityChannel) {
+    return Format.createTextSampleFormat(
+        adaptationSetId
+            + ":cea608"
+            + (accessibilityChannel != Format.NO_VALUE ? ":" + accessibilityChannel : ""),
+        MimeTypes.APPLICATION_CEA608,
+        /* codecs= */ null,
+        /* bitrate= */ Format.NO_VALUE,
+        /* selectionFlags= */ 0,
+        language,
+        accessibilityChannel,
+        /* drmInitData= */ null,
+        Format.OFFSET_SAMPLE_RELATIVE,
+        /* initializationData= */ null);
  }
  @SuppressWarnings("unchecked")
@@ -761,7 +836,7 @@ import java.util.List;
          primaryTrackGroupIndex,
          embeddedEventMessageTrackGroupIndex,
          embeddedCea608TrackGroupIndex,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
    }
    public static TrackGroupInfo embeddedEmsgTrack(int[] adaptationSetIndices,
@@ -773,7 +848,7 @@ import java.util.List;
          primaryTrackGroupIndex,
          C.INDEX_UNSET,
          C.INDEX_UNSET,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
    }
    public static TrackGroupInfo embeddedCea608Track(int[] adaptationSetIndices,
@@ -785,7 +860,7 @@ import java.util.List;
          primaryTrackGroupIndex,
          C.INDEX_UNSET,
          C.INDEX_UNSET,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
    }
    public static TrackGroupInfo mpdEventTrack(int eventStreamIndex) {
@@ -793,7 +868,7 @@ import java.util.List;
          C.TRACK_TYPE_METADATA,
          CATEGORY_MANIFEST_EVENTS,
          new int[0],
-          -1,
+          /* primaryTrackGroupIndex= */ -1,
          C.INDEX_UNSET,
          C.INDEX_UNSET,
          eventStreamIndex);

--- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java
+++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java
@@ -54,7 +54,6 @@ import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 /**
@@ -86,7 +85,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
        int trackType,
        long elapsedRealtimeOffsetMs,
        boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
        @Nullable PlayerTrackEmsgHandler playerEmsgHandler,
        @Nullable TransferListener transferListener) {
      DataSource dataSource = dataSourceFactory.createDataSource();
@@ -104,7 +103,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
          elapsedRealtimeOffsetMs,
          maxSegmentsPerLoad,
          enableEventMessageTrack,
-          enableCea608Track,
+          closedCaptionFormats,
          playerEmsgHandler);
    }
@@ -141,9 +140,8 @@ public class DefaultDashChunkSource implements DashChunkSource {
   * @param maxSegmentsPerLoad The maximum number of segments to combine into a single request. Note
   *     that segments will only be combined if their {@link Uri}s are the same and if their data
   *     ranges are adjacent.
-   * @param enableEventMessageTrack Whether the chunks generated by the source may output an event
+   * @param enableEventMessageTrack Whether to output an event message track.
-   *     message track.
+   * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output.
-   * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608 track.
   * @param playerTrackEmsgHandler The {@link PlayerTrackEmsgHandler} instance to handle emsg
   *     messages targeting the player. Maybe null if this is not necessary.
   */
@@ -158,7 +156,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
      long elapsedRealtimeOffsetMs,
      int maxSegmentsPerLoad,
      boolean enableEventMessageTrack,
-      boolean enableCea608Track,
+      List<Format> closedCaptionFormats,
      @Nullable PlayerTrackEmsgHandler playerTrackEmsgHandler) {
    this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
    this.manifest = manifest;
@@ -184,7 +182,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
              trackType,
              representation,
              enableEventMessageTrack,
-              enableCea608Track,
+              closedCaptionFormats,
              playerTrackEmsgHandler);
    }
  }
@@ -629,7 +627,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
        int trackType,
        Representation representation,
        boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
        TrackOutput playerEmsgTrackOutput) {
      this(
          periodDurationUs,
@@ -638,7 +636,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
              trackType,
              representation,
              enableEventMessageTrack,
-              enableCea608Track,
+              closedCaptionFormats,
              playerEmsgTrackOutput),
          /* segmentNumShift= */ 0,
          representation.getIndex());
@@ -783,7 +781,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
        int trackType,
        Representation representation,
        boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
        TrackOutput playerEmsgTrackOutput) {
      String containerMimeType = representation.format.containerMimeType;
      if (mimeTypeIsRawText(containerMimeType)) {
@@ -799,12 +797,6 @@ public class DefaultDashChunkSource implements DashChunkSource {
        if (enableEventMessageTrack) {
          flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
        }
-        // TODO: Use caption format information from the manifest if available.
-        List<Format> closedCaptionFormats =
-            enableCea608Track
-                ? Collections.singletonList(
-                    Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0, null))
-                : Collections.emptyList();
        extractor =
            new FragmentedMp4Extractor(
                flags, null, null, null, closedCaptionFormats, playerEmsgTrackOutput);