Fix DRM protected SmoothStreaming with subtitles

Issue: #5378
PiperOrigin-RevId: 229261658

RELEASENOTES.md

@@ -27,6 +27,8 @@
   * Rename TaskState to DownloadState.
   * Add new states to DownloadState.
   * Replace DownloadState.action with DownloadAction fields.
+* SmoothStreaming: Fix support for subtitles in DRM protected streams
+  ([#5378](https://github.com/google/ExoPlayer/issues/5378)).
 * Add support for SHOUTcast ICY metadata
   ([#3735](https://github.com/google/ExoPlayer/issues/3735)).
 * IMA extension:

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/DefaultSsChunkSource.java

@@ -61,14 +61,13 @@ public class DefaultSsChunkSource implements SsChunkSource {
         SsManifest manifest,
         int elementIndex,
         TrackSelection trackSelection,
-        TrackEncryptionBox[] trackEncryptionBoxes,
         @Nullable TransferListener transferListener) {
       DataSource dataSource = dataSourceFactory.createDataSource();
       if (transferListener != null) {
         dataSource.addTransferListener(transferListener);
       }
-      return new DefaultSsChunkSource(manifestLoaderErrorThrower, manifest, elementIndex,
-          trackSelection, dataSource, trackEncryptionBoxes);
+      return new DefaultSsChunkSource(
+          manifestLoaderErrorThrower, manifest, elementIndex, trackSelection, dataSource);
     }
 
   }
@@ -90,15 +89,13 @@ public class DefaultSsChunkSource implements SsChunkSource {
    * @param streamElementIndex The index of the stream element in the manifest.
    * @param trackSelection The track selection.
    * @param dataSource A {@link DataSource} suitable for loading the media data.
-   * @param trackEncryptionBoxes Track encryption boxes for the stream.
    */
   public DefaultSsChunkSource(
       LoaderErrorThrower manifestLoaderErrorThrower,
       SsManifest manifest,
       int streamElementIndex,
       TrackSelection trackSelection,
-      DataSource dataSource,
-      TrackEncryptionBox[] trackEncryptionBoxes) {
+      DataSource dataSource) {
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
     this.manifest = manifest;
     this.streamElementIndex = streamElementIndex;
@@ -110,6 +107,8 @@ public class DefaultSsChunkSource implements SsChunkSource {
     for (int i = 0; i < extractorWrappers.length; i++) {
       int manifestTrackIndex = trackSelection.getIndexInTrackGroup(i);
       Format format = streamElement.formats[manifestTrackIndex];
+      TrackEncryptionBox[] trackEncryptionBoxes =
+          format.drmInitData != null ? manifest.protectionElement.trackEncryptionBoxes : null;
       int nalUnitLengthFieldLength = streamElement.type == C.TRACK_TYPE_VIDEO ? 4 : 0;
       Track track = new Track(manifestTrackIndex, streamElement.type, streamElement.timescale,
           C.TIME_UNSET, manifest.durationUs, format, Track.TRANSFORMATION_NONE,

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsChunkSource.java

@@ -16,7 +16,6 @@
 package com.google.android.exoplayer2.source.smoothstreaming;
 
 import android.support.annotation.Nullable;
-import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.source.chunk.ChunkSource;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
@@ -38,7 +37,6 @@ public interface SsChunkSource extends ChunkSource {
      * @param manifest The initial manifest.
      * @param streamElementIndex The index of the corresponding stream element in the manifest.
      * @param trackSelection The track selection.
-     * @param trackEncryptionBoxes Track encryption boxes for the stream.
      * @param transferListener The transfer listener which should be informed of any data transfers.
      *     May be null if no listener is available.
      * @return The created {@link SsChunkSource}.
@@ -48,7 +46,6 @@ public interface SsChunkSource extends ChunkSource {
         SsManifest manifest,
         int streamElementIndex,
         TrackSelection trackSelection,
-        TrackEncryptionBox[] trackEncryptionBoxes,
         @Nullable TransferListener transferListener);
   }
 

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/SsMediaPeriod.java

@@ -16,10 +16,8 @@
 package com.google.android.exoplayer2.source.smoothstreaming;
 
 import android.support.annotation.Nullable;
-import android.util.Base64;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.SeekParameters;
-import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.offline.StreamKey;
 import com.google.android.exoplayer2.source.CompositeSequenceableLoaderFactory;
 import com.google.android.exoplayer2.source.MediaPeriod;
@@ -30,7 +28,6 @@ import com.google.android.exoplayer2.source.TrackGroup;
 import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.source.chunk.ChunkSampleStream;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest;
-import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.ProtectionElement;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.upstream.Allocator;
 import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy;
@@ -44,8 +41,6 @@ import java.util.List;
 /* package */ final class SsMediaPeriod
     implements MediaPeriod, SequenceableLoader.Callback<ChunkSampleStream<SsChunkSource>> {
 
-  private static final int INITIALIZATION_VECTOR_SIZE = 8;
-
   private final SsChunkSource.Factory chunkSourceFactory;
   private final @Nullable TransferListener transferListener;
   private final LoaderErrorThrower manifestLoaderErrorThrower;
@@ -53,7 +48,6 @@ import java.util.List;
   private final EventDispatcher eventDispatcher;
   private final Allocator allocator;
   private final TrackGroupArray trackGroups;
-  private final TrackEncryptionBox[] trackEncryptionBoxes;
   private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
 
   private @Nullable Callback callback;
@@ -71,6 +65,7 @@ import java.util.List;
       EventDispatcher eventDispatcher,
       LoaderErrorThrower manifestLoaderErrorThrower,
       Allocator allocator) {
+    this.manifest = manifest;
     this.chunkSourceFactory = chunkSourceFactory;
     this.transferListener = transferListener;
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
@@ -78,18 +73,7 @@ import java.util.List;
     this.eventDispatcher = eventDispatcher;
     this.allocator = allocator;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
-
     trackGroups = buildTrackGroups(manifest);
-    ProtectionElement protectionElement = manifest.protectionElement;
-    if (protectionElement != null) {
-      byte[] keyId = getProtectionElementKeyId(protectionElement.data);
-      // We assume pattern encryption does not apply.
-      trackEncryptionBoxes = new TrackEncryptionBox[] {
-          new TrackEncryptionBox(true, null, INITIALIZATION_VECTOR_SIZE, keyId, 0, 0, null)};
-    } else {
-      trackEncryptionBoxes = null;
-    }
-    this.manifest = manifest;
     sampleStreams = newSampleStreamArray(0);
     compositeSequenceableLoader =
         compositeSequenceableLoaderFactory.createCompositeSequenceableLoader(sampleStreams);
@@ -241,7 +225,6 @@ import java.util.List;
             manifest,
             streamElementIndex,
             selection,
-            trackEncryptionBoxes,
             transferListener);
     return new ChunkSampleStream<>(
         manifest.streamElements[streamElementIndex].type,
@@ -267,26 +250,4 @@ import java.util.List;
   private static ChunkSampleStream<SsChunkSource>[] newSampleStreamArray(int length) {
     return new ChunkSampleStream[length];
   }
-
-  private static byte[] getProtectionElementKeyId(byte[] initData) {
-    StringBuilder initDataStringBuilder = new StringBuilder();
-    for (int i = 0; i < initData.length; i += 2) {
-      initDataStringBuilder.append((char) initData[i]);
-    }
-    String initDataString = initDataStringBuilder.toString();
-    String keyIdString = initDataString.substring(
-        initDataString.indexOf("<KID>") + 5, initDataString.indexOf("</KID>"));
-    byte[] keyId = Base64.decode(keyIdString, Base64.DEFAULT);
-    swap(keyId, 0, 3);
-    swap(keyId, 1, 2);
-    swap(keyId, 4, 5);
-    swap(keyId, 6, 7);
-    return keyId;
-  }
-
-  private static void swap(byte[] data, int firstPosition, int secondPosition) {
-    byte temp = data[firstPosition];
-    data[firstPosition] = data[secondPosition];
-    data[secondPosition] = temp;
-  }
 }

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifest.java

@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.source.smoothstreaming.manifest;
 import android.net.Uri;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.offline.FilterableManifest;
 import com.google.android.exoplayer2.offline.StreamKey;
 import com.google.android.exoplayer2.util.Assertions;
@@ -41,10 +42,12 @@ public class SsManifest implements FilterableManifest<SsManifest> {
 
     public final UUID uuid;
     public final byte[] data;
+    public final TrackEncryptionBox[] trackEncryptionBoxes;
 
-    public ProtectionElement(UUID uuid, byte[] data) {
+    public ProtectionElement(UUID uuid, byte[] data, TrackEncryptionBox[] trackEncryptionBoxes) {
       this.uuid = uuid;
       this.data = data;
+      this.trackEncryptionBoxes = trackEncryptionBoxes;
     }
   }
 

library/smoothstreaming/src/main/java/com/google/android/exoplayer2/source/smoothstreaming/manifest/SsManifestParser.java

@@ -25,6 +25,7 @@ import com.google.android.exoplayer2.ParserException;
 import com.google.android.exoplayer2.drm.DrmInitData;
 import com.google.android.exoplayer2.drm.DrmInitData.SchemeData;
 import com.google.android.exoplayer2.extractor.mp4.PsshAtomUtil;
+import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.ProtectionElement;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.StreamElement;
 import com.google.android.exoplayer2.upstream.ParsingLoadable;
@@ -397,9 +398,10 @@ public class SsManifestParser implements ParsingLoadable.Parser<SsManifest> {
 
     public static final String TAG = "Protection";
     public static final String TAG_PROTECTION_HEADER = "ProtectionHeader";
-
     public static final String KEY_SYSTEM_ID = "SystemID";
 
+    private static final int INITIALIZATION_VECTOR_SIZE = 8;
+
     private boolean inProtectionHeader;
     private UUID uuid;
     private byte[] initData;
@@ -439,7 +441,44 @@ public class SsManifestParser implements ParsingLoadable.Parser<SsManifest> {
 
     @Override
     public Object build() {
-      return new ProtectionElement(uuid, PsshAtomUtil.buildPsshAtom(uuid, initData));
+      return new ProtectionElement(
+          uuid, PsshAtomUtil.buildPsshAtom(uuid, initData), buildTrackEncryptionBoxes(initData));
+    }
+
+    private static TrackEncryptionBox[] buildTrackEncryptionBoxes(byte[] initData) {
+      return new TrackEncryptionBox[] {
+        new TrackEncryptionBox(
+            /* isEncrypted= */ true,
+            /* schemeType= */ null,
+            INITIALIZATION_VECTOR_SIZE,
+            getProtectionElementKeyId(initData),
+            /* defaultEncryptedBlocks= */ 0,
+            /* defaultClearBlocks= */ 0,
+            /* defaultInitializationVector= */ null)
+      };
+    }
+
+    private static byte[] getProtectionElementKeyId(byte[] initData) {
+      StringBuilder initDataStringBuilder = new StringBuilder();
+      for (int i = 0; i < initData.length; i += 2) {
+        initDataStringBuilder.append((char) initData[i]);
+      }
+      String initDataString = initDataStringBuilder.toString();
+      String keyIdString =
+          initDataString.substring(
+              initDataString.indexOf("<KID>") + 5, initDataString.indexOf("</KID>"));
+      byte[] keyId = Base64.decode(keyIdString, Base64.DEFAULT);
+      swap(keyId, 0, 3);
+      swap(keyId, 1, 2);
+      swap(keyId, 4, 5);
+      swap(keyId, 6, 7);
+      return keyId;
+    }
+
+    private static void swap(byte[] data, int firstPosition, int secondPosition) {
+      byte temp = data[firstPosition];
+      data[firstPosition] = data[secondPosition];
+      data[secondPosition] = temp;
     }
 
     private static String stripCurlyBraces(String uuidString) {

library/smoothstreaming/src/test/assets/sample_ismc_1

@@ -3,7 +3,7 @@
     Duration="2300000000" TimeScale="10000000">
     <Protection>
         <ProtectionHeader SystemID="9A04F079-9840-4286-AB92-E65BE0885F95">
-            <!-- Base 64-Encoded data omitted for clarity -->
+            fgMAAAEAAQB0AzwAVwBSAE0ASABFAEEARABFAFIAIAB4AG0AbABuAHMAPQAiAGgAdAB0AHAAOgAvAC8AcwBjAGgAZQBtAGEAcwAuAG0AaQBjAHIAbwBzAG8AZgB0AC4AYwBvAG0ALwBEAFIATQAvADIAMAAwADcALwAwADMALwBQAGwAYQB5AFIAZQBhAGQAeQBIAGUAYQBkAGUAcgAiACAAdgBlAHIAcwBpAG8AbgA9ACIANAAuADAALgAwAC4AMAAiAD4APABEAEEAVABBAD4APABQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsARQBZAEwARQBOAD4AMQA2ADwALwBLAEUAWQBMAEUATgA+ADwAQQBMAEcASQBEAD4AQQBFAFMAQwBUAFIAPAAvAEEATABHAEkARAA+ADwALwBQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsASQBEAD4AQgBhAFUATQBPAEcAYwBzAGgAVQBDAEQAZAB3ADMANABZAGMAawBmAFoAQQA9AD0APAAvAEsASQBEAD4APABDAEgARQBDAEsAUwBVAE0APgBnADcATgBhAFIARABJAEkATwA5ADAAPQA8AC8AQwBIAEUAQwBLAFMAVQBNAD4APABMAEEAXwBVAFIATAA+AGgAdAB0AHAAcwA6AC8ALwBUAC0ATwBOAEwASQBOAEUALgBEAFUATQBNAFkALQBTAEUAUgBWAEUAUgAvAEEAcgB0AGUAbQBpAHMATABpAGMAZQBuAHMAZQBTAGUAcgB2AGUAcgAvAFAAbABhAHkAUgBlAGEAZAB5AE0AYQBuAGEAZwBlAHIALgBhAHMAbQB4ADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwAQwBJAEQAPgAxADcANQA4ADIANgA8AC8AQwBJAEQAPgA8AEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4ANwAuADEALgAxADUANgA1AC4ANAA8AC8ASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA8AC8AQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwALwBEAEEAVABBAD4APAAvAFcAUgBNAEgARQBBAEQARQBSAD4A
         </ProtectionHeader>
     </Protection>
 

library/smoothstreaming/src/test/assets/sample_ismc_2

@@ -3,7 +3,7 @@
     Duration="2300000000" TimeScale="10000000">
     <Protection>
         <ProtectionHeader SystemID="{9A04F079-9840-4286-AB92-E65BE0885F95}">
-            <!-- Base 64-Encoded data omitted for clarity -->
+            fgMAAAEAAQB0AzwAVwBSAE0ASABFAEEARABFAFIAIAB4AG0AbABuAHMAPQAiAGgAdAB0AHAAOgAvAC8AcwBjAGgAZQBtAGEAcwAuAG0AaQBjAHIAbwBzAG8AZgB0AC4AYwBvAG0ALwBEAFIATQAvADIAMAAwADcALwAwADMALwBQAGwAYQB5AFIAZQBhAGQAeQBIAGUAYQBkAGUAcgAiACAAdgBlAHIAcwBpAG8AbgA9ACIANAAuADAALgAwAC4AMAAiAD4APABEAEEAVABBAD4APABQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsARQBZAEwARQBOAD4AMQA2ADwALwBLAEUAWQBMAEUATgA+ADwAQQBMAEcASQBEAD4AQQBFAFMAQwBUAFIAPAAvAEEATABHAEkARAA+ADwALwBQAFIATwBUAEUAQwBUAEkATgBGAE8APgA8AEsASQBEAD4AQgBhAFUATQBPAEcAYwBzAGgAVQBDAEQAZAB3ADMANABZAGMAawBmAFoAQQA9AD0APAAvAEsASQBEAD4APABDAEgARQBDAEsAUwBVAE0APgBnADcATgBhAFIARABJAEkATwA5ADAAPQA8AC8AQwBIAEUAQwBLAFMAVQBNAD4APABMAEEAXwBVAFIATAA+AGgAdAB0AHAAcwA6AC8ALwBUAC0ATwBOAEwASQBOAEUALgBEAFUATQBNAFkALQBTAEUAUgBWAEUAUgAvAEEAcgB0AGUAbQBpAHMATABpAGMAZQBuAHMAZQBTAGUAcgB2AGUAcgAvAFAAbABhAHkAUgBlAGEAZAB5AE0AYQBuAGEAZwBlAHIALgBhAHMAbQB4ADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwAQwBJAEQAPgAxADcANQA4ADIANgA8AC8AQwBJAEQAPgA8AEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4ANwAuADEALgAxADUANgA1AC4ANAA8AC8ASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA8AC8AQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwALwBEAEEAVABBAD4APAAvAFcAUgBNAEgARQBBAEQARQBSAD4A
         </ProtectionHeader>
     </Protection>
 

library/smoothstreaming/src/test/java/com/google/android/exoplayer2/source/smoothstreaming/SsTestUtils.java

@@ -15,13 +15,12 @@
  */
 package com.google.android.exoplayer2.source.smoothstreaming;
 
-import android.util.Base64;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.extractor.mp4.TrackEncryptionBox;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.ProtectionElement;
 import com.google.android.exoplayer2.source.smoothstreaming.manifest.SsManifest.StreamElement;
-import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 
 /** Util methods for SmoothStreaming tests. */
@@ -41,12 +40,7 @@ public class SsTestUtils {
   private static final int TEST_MAX_HEIGHT = 768;
   private static final String TEST_LANGUAGE = "eng";
   private static final ProtectionElement TEST_PROTECTION_ELEMENT =
-      new ProtectionElement(
-          C.WIDEVINE_UUID,
-          ("<KID>"
-                  + Base64.encodeToString(new byte[] {0, 1, 2, 3, 4, 5, 6, 7}, Base64.DEFAULT)
-                  + "</KID>")
-              .getBytes(StandardCharsets.UTF_16LE));
+      new ProtectionElement(C.WIDEVINE_UUID, new byte[0], new TrackEncryptionBox[0]);
 
   private SsTestUtils() {}