Enable ID3-in-EMSG for HLS streams

This supports both chunkless & traditional preparation

PiperOrigin-RevId: 273938344

RELEASENOTES.md

@@ -7,6 +7,8 @@
   ([#6203](https://github.com/google/ExoPlayer/issues/6203)).
 * DASH: Support `Label` elements
   ([#6297](https://github.com/google/ExoPlayer/issues/6297)).
+* HLS: Add support for ID3 in EMSG when using FMP4 streams
+  ([spec](https://aomediacodec.github.io/av1-id3/)).
 * Metadata: Expose the raw ICY metadata through `IcyInfo`
   ([#6476](https://github.com/google/ExoPlayer/issues/6476)).
 * UI

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/DefaultHlsExtractorFactory.java

@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.extractor.ts.Ac4Extractor;
 import com.google.android.exoplayer2.extractor.ts.AdtsExtractor;
 import com.google.android.exoplayer2.extractor.ts.DefaultTsPayloadReaderFactory;
 import com.google.android.exoplayer2.extractor.ts.TsExtractor;
+import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TimestampAdjuster;
 import java.io.EOFException;
@@ -158,7 +159,7 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
 
     if (!(extractorByFileExtension instanceof FragmentedMp4Extractor)) {
       FragmentedMp4Extractor fragmentedMp4Extractor =
-          createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
+          createFragmentedMp4Extractor(timestampAdjuster, format, drmInitData, muxedCaptionFormats);
       if (sniffQuietly(fragmentedMp4Extractor, extractorInput)) {
         return buildResult(fragmentedMp4Extractor);
       }
@@ -208,7 +209,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
         || lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)
         || lastPathSegment.startsWith(MP4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)
         || lastPathSegment.startsWith(CMF_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)) {
-      return createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
+      return createFragmentedMp4Extractor(
+          timestampAdjuster, format, drmInitData, muxedCaptionFormats);
     } else {
       // For any other file extension, we assume TS format.
       return createTsExtractor(
@@ -267,10 +269,21 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
 
   private static FragmentedMp4Extractor createFragmentedMp4Extractor(
       TimestampAdjuster timestampAdjuster,
+      Format format,
       DrmInitData drmInitData,
       @Nullable List<Format> muxedCaptionFormats) {
+    boolean isVariant = false;
+    for (int i = 0; i < format.metadata.length(); i++) {
+      Metadata.Entry entry = format.metadata.get(i);
+      if (entry instanceof HlsTrackMetadataEntry) {
+        isVariant = !((HlsTrackMetadataEntry) entry).variantInfos.isEmpty();
+        break;
+      }
+    }
+    // Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid
+    // creating a separate EMSG track for every audio track in a video stream.
     return new FragmentedMp4Extractor(
-        /* flags= */ 0,
+        /* flags= */ isVariant ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0,
         timestampAdjuster,
         /* sideloadedTrack= */ null,
         drmInitData,

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java

@@ -71,6 +71,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
   private final TimestampAdjusterProvider timestampAdjusterProvider;
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
   private final boolean allowChunklessPreparation;
+  private final @HlsMetadataType int metadataType;
   private final boolean useSessionKeys;
 
   private @Nullable Callback callback;
@@ -110,6 +111,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
       Allocator allocator,
       CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
       boolean allowChunklessPreparation,
+      @HlsMetadataType int metadataType,
       boolean useSessionKeys) {
     this.extractorFactory = extractorFactory;
     this.playlistTracker = playlistTracker;
@@ -120,6 +122,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
     this.allocator = allocator;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
     this.allowChunklessPreparation = allowChunklessPreparation;
+    this.metadataType = metadataType;
     this.useSessionKeys = useSessionKeys;
     compositeSequenceableLoader =
         compositeSequenceableLoaderFactory.createCompositeSequenceableLoader();
@@ -736,7 +739,8 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
         positionUs,
         muxedAudioFormat,
         loadErrorHandlingPolicy,
-        eventDispatcher);
+        eventDispatcher,
+        metadataType);
   }
 
   private static Map<String, DrmInitData> deriveOverridingDrmInitData(

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaSource.java

@@ -67,6 +67,7 @@ public final class HlsMediaSource extends BaseMediaSource
     private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
     private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
     private boolean allowChunklessPreparation;
+    @HlsMetadataType private int metadataType;
     private boolean useSessionKeys;
     private boolean isCreateCalled;
     @Nullable private Object tag;
@@ -95,6 +96,7 @@ public final class HlsMediaSource extends BaseMediaSource
       extractorFactory = HlsExtractorFactory.DEFAULT;
       loadErrorHandlingPolicy = new DefaultLoadErrorHandlingPolicy();
       compositeSequenceableLoaderFactory = new DefaultCompositeSequenceableLoaderFactory();
+      metadataType = HlsMetadataType.ID3;
     }
 
     /**
@@ -238,6 +240,31 @@ public final class HlsMediaSource extends BaseMediaSource
     }
 
     /**
+     * Sets the type of metadata to extract from the HLS source (defaults to {@link
+     * HlsMetadataType#ID3}).
+     *
+     * <p>HLS supports in-band ID3 in both TS and fMP4 streams, but in the fMP4 case the data is
+     * wrapped in an EMSG box [<a href="https://aomediacodec.github.io/av1-id3/">spec</a>].
+     *
+     * <p>If this is set to {@link HlsMetadataType#ID3} then raw ID3 metadata of will be extracted
+     * from TS sources. From fMP4 streams EMSGs containing metadata of this type (in the variant
+     * stream only) will be unwrapped to expose the inner data. All other in-band metadata will be
+     * dropped.
+     *
+     * <p>If this is set to {@link HlsMetadataType#EMSG} then all EMSG data from the fMP4 variant
+     * stream will be extracted. No metadata will be extracted from TS streams, since they don't
+     * support EMSG.
+     *
+     * @param metadataType The type of metadata to extract.
+     * @return This factory, for convenience.
+     */
+    public Factory setMetadataType(@HlsMetadataType int metadataType) {
+      Assertions.checkState(!isCreateCalled);
+      this.metadataType = metadataType;
+      return this;
+    }
+
+    /**
      * Sets whether to use #EXT-X-SESSION-KEY tags provided in the master playlist. If enabled, it's
      * assumed that any single session key declared in the master playlist can be used to obtain all
      * of the keys required for playback. For media where this is not true, this option should not
@@ -272,6 +299,7 @@ public final class HlsMediaSource extends BaseMediaSource
           playlistTrackerFactory.createTracker(
               hlsDataSourceFactory, loadErrorHandlingPolicy, playlistParserFactory),
           allowChunklessPreparation,
+          metadataType,
           useSessionKeys,
           tag);
     }
@@ -305,6 +333,7 @@ public final class HlsMediaSource extends BaseMediaSource
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
   private final LoadErrorHandlingPolicy loadErrorHandlingPolicy;
   private final boolean allowChunklessPreparation;
+  private final @HlsMetadataType int metadataType;
   private final boolean useSessionKeys;
   private final HlsPlaylistTracker playlistTracker;
   private final @Nullable Object tag;
@@ -319,6 +348,7 @@ public final class HlsMediaSource extends BaseMediaSource
       LoadErrorHandlingPolicy loadErrorHandlingPolicy,
       HlsPlaylistTracker playlistTracker,
       boolean allowChunklessPreparation,
+      @HlsMetadataType int metadataType,
       boolean useSessionKeys,
       @Nullable Object tag) {
     this.manifestUri = manifestUri;
@@ -328,6 +358,7 @@ public final class HlsMediaSource extends BaseMediaSource
     this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
     this.playlistTracker = playlistTracker;
     this.allowChunklessPreparation = allowChunklessPreparation;
+    this.metadataType = metadataType;
     this.useSessionKeys = useSessionKeys;
     this.tag = tag;
   }
@@ -363,6 +394,7 @@ public final class HlsMediaSource extends BaseMediaSource
         allocator,
         compositeSequenceableLoaderFactory,
         allowChunklessPreparation,
+        metadataType,
         useSessionKeys);
   }
 

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMetadataType.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.google.android.exoplayer2.source.hls;
+
+import static java.lang.annotation.RetentionPolicy.SOURCE;
+
+import androidx.annotation.IntDef;
+import java.lang.annotation.Retention;
+
+/**
+ * The types of metadata that can be extracted from HLS streams.
+ *
+ * <p>See {@link HlsMediaSource.Factory#setMetadataType(int)}.
+ */
+@Retention(SOURCE)
+@IntDef({HlsMetadataType.ID3, HlsMetadataType.EMSG})
+public @interface HlsMetadataType {
+  int ID3 = 1;
+  int EMSG = 3;
+}

library/hls/src/test/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriodTest.java

@@ -91,6 +91,7 @@ public final class HlsMediaPeriodTest {
               mock(Allocator.class),
               mock(CompositeSequenceableLoaderFactory.class),
               /* allowChunklessPreparation =*/ true,
+              HlsMetadataType.ID3,
               /* useSessionKeys= */ false);
         };