Support unwrapping nested Metadata messages in MetadataRenderer

Initially this supports ID3-in-EMSG, but can also be used to support SCTE35-in-EMSG too.

PiperOrigin-RevId: 263535925

library/core/src/main/java/com/google/android/exoplayer2/metadata/Metadata.java

@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.metadata;
 import android.os.Parcel;
 import android.os.Parcel;
 import android.os.Parcelable;
 import android.os.Parcelable;
 import androidx.annotation.Nullable;
 import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.util.Util;
 import java.util.Arrays;
 import java.util.Arrays;
 import java.util.List;
 import java.util.List;
@@ -28,10 +29,27 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
  */
  */
 public final class Metadata implements Parcelable {
 public final class Metadata implements Parcelable {
 
 
+  /** A metadata entry. */
+  public interface Entry extends Parcelable {
+
     /**
     /**
-   * A metadata entry.
+     * Returns the {@link Format} that can be used to decode the wrapped metadata in {@link
+     * #getWrappedMetadataBytes()}, or null if this Entry doesn't contain wrapped metadata.
      */
      */
-  public interface Entry extends Parcelable {}
+    @Nullable
+    default Format getWrappedMetadataFormat() {
+      return null;
+    }
+
+    /**
+     * Returns the bytes of the wrapped metadata in this Entry, or null if it doesn't contain
+     * wrapped metadata.
+     */
+    @Nullable
+    default byte[] getWrappedMetadataBytes() {
+      return null;
+    }
+  }
 
 
   private final Entry[] entries;
   private final Entry[] entries;
 
 

library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java

@@ -27,7 +27,9 @@ import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.FormatHolder;
 import com.google.android.exoplayer2.FormatHolder;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.util.Util;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Arrays;
+import java.util.List;
 
 
 /**
 /**
  * A renderer for metadata.
  * A renderer for metadata.
@@ -129,14 +131,20 @@ public final class MetadataRenderer extends BaseRenderer implements Callback {
         } else {
         } else {
           buffer.subsampleOffsetUs = subsampleOffsetUs;
           buffer.subsampleOffsetUs = subsampleOffsetUs;
           buffer.flip();
           buffer.flip();
-          int index = (pendingMetadataIndex + pendingMetadataCount) % MAX_PENDING_METADATA_COUNT;
           Metadata metadata = decoder.decode(buffer);
           Metadata metadata = decoder.decode(buffer);
           if (metadata != null) {
           if (metadata != null) {
-            pendingMetadata[index] = metadata;
+            List<Metadata.Entry> entries = new ArrayList<>(metadata.length());
+            decodeWrappedMetadata(metadata, entries);
+            if (!entries.isEmpty()) {
+              Metadata expandedMetadata = new Metadata(entries);
+              int index =
+                  (pendingMetadataIndex + pendingMetadataCount) % MAX_PENDING_METADATA_COUNT;
+              pendingMetadata[index] = expandedMetadata;
               pendingMetadataTimestamps[index] = buffer.timeUs;
               pendingMetadataTimestamps[index] = buffer.timeUs;
               pendingMetadataCount++;
               pendingMetadataCount++;
             }
             }
           }
           }
+        }
       } else if (result == C.RESULT_FORMAT_READ) {
       } else if (result == C.RESULT_FORMAT_READ) {
         subsampleOffsetUs = formatHolder.format.subsampleOffsetUs;
         subsampleOffsetUs = formatHolder.format.subsampleOffsetUs;
       }
       }
@@ -150,6 +158,36 @@ public final class MetadataRenderer extends BaseRenderer implements Callback {
     }
     }
   }
   }
 
 
+  /**
+   * Iterates through {@code metadata.entries} and checks each one to see if contains wrapped
+   * metadata. If it does, then we recursively decode the wrapped metadata. If it doesn't (recursion
+   * base-case), we add the {@link Metadata.Entry} to {@code decodedEntries} (output parameter).
+   */
+  private void decodeWrappedMetadata(Metadata metadata, List<Metadata.Entry> decodedEntries) {
+    for (int i = 0; i < metadata.length(); i++) {
+      Format wrappedMetadataFormat = metadata.get(i).getWrappedMetadataFormat();
+      if (wrappedMetadataFormat != null && decoderFactory.supportsFormat(wrappedMetadataFormat)) {
+        MetadataDecoder wrappedMetadataDecoder =
+            decoderFactory.createDecoder(wrappedMetadataFormat);
+        // wrappedMetadataFormat != null so wrappedMetadataBytes must be non-null too.
+        byte[] wrappedMetadataBytes =
+            Assertions.checkNotNull(metadata.get(i).getWrappedMetadataBytes());
+        buffer.clear();
+        buffer.ensureSpaceForWrite(wrappedMetadataBytes.length);
+        buffer.data.put(wrappedMetadataBytes);
+        buffer.flip();
+        @Nullable Metadata innerMetadata = wrappedMetadataDecoder.decode(buffer);
+        if (innerMetadata != null) {
+          // The decoding succeeded, so we'll try another level of unwrapping.
+          decodeWrappedMetadata(innerMetadata, decodedEntries);
+        }
+      } else {
+        // Entry doesn't contain any wrapped metadata, so output it directly.
+        decodedEntries.add(metadata.get(i));
+      }
+    }
+  }
+
   @Override
   @Override
   protected void onDisabled() {
   protected void onDisabled() {
     flushPendingMetadata();
     flushPendingMetadata();

library/core/src/main/java/com/google/android/exoplayer2/metadata/emsg/EventMessage.java

@@ -20,7 +20,10 @@ import static com.google.android.exoplayer2.util.Util.castNonNull;
 import android.os.Parcel;
 import android.os.Parcel;
 import android.os.Parcelable;
 import android.os.Parcelable;
 import androidx.annotation.Nullable;
 import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.util.Util;
 import java.util.Arrays;
 import java.util.Arrays;
 
 
@@ -29,6 +32,13 @@ import java.util.Arrays;
  */
  */
 public final class EventMessage implements Metadata.Entry {
 public final class EventMessage implements Metadata.Entry {
 
 
+  @VisibleForTesting
+  public static final String ID3_SCHEME_ID = "https://developer.apple.com/streaming/emsg-id3";
+
+  private static final Format ID3_FORMAT =
+      Format.createSampleFormat(
+          /* id= */ null, MimeTypes.APPLICATION_ID3, Format.OFFSET_SAMPLE_RELATIVE);
+
   /**
   /**
    * The message scheme.
    * The message scheme.
    */
    */
@@ -82,6 +92,18 @@ public final class EventMessage implements Metadata.Entry {
   }
   }
 
 
   @Override
   @Override
+  @Nullable
+  public Format getWrappedMetadataFormat() {
+    return ID3_SCHEME_ID.equals(schemeIdUri) ? ID3_FORMAT : null;
+  }
+
+  @Override
+  @Nullable
+  public byte[] getWrappedMetadataBytes() {
+    return ID3_SCHEME_ID.equals(schemeIdUri) ? messageData : null;
+  }
+
+  @Override
   public int hashCode() {
   public int hashCode() {
     if (hashCode == 0) {
     if (hashCode == 0) {
       int result = 17;
       int result = 17;

library/core/src/test/java/com/google/android/exoplayer2/metadata/MetadataRendererTest.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.google.android.exoplayer2.metadata;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.ExoPlaybackException;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.metadata.emsg.EventMessage;
+import com.google.android.exoplayer2.metadata.emsg.EventMessageEncoder;
+import com.google.android.exoplayer2.metadata.id3.TextInformationFrame;
+import com.google.android.exoplayer2.testutil.FakeSampleStream;
+import com.google.android.exoplayer2.testutil.TestUtil;
+import com.google.android.exoplayer2.util.Assertions;
+import com.google.android.exoplayer2.util.MimeTypes;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link MetadataRenderer}. */
+@RunWith(AndroidJUnit4.class)
+public class MetadataRendererTest {
+
+  private static final Format EMSG_FORMAT =
+      Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG, Format.OFFSET_SAMPLE_RELATIVE);
+
+  private final EventMessageEncoder eventMessageEncoder = new EventMessageEncoder();
+
+  @Test
+  public void decodeMetadata() throws Exception {
+    EventMessage emsg =
+        new EventMessage(
+            "urn:test-scheme-id",
+            /* value= */ "",
+            /* durationMs= */ 1,
+            /* id= */ 0,
+            "Test data".getBytes(UTF_8));
+
+    List<Metadata> metadata = runRenderer(eventMessageEncoder.encode(emsg));
+
+    assertThat(metadata).hasSize(1);
+    assertThat(metadata.get(0).length()).isEqualTo(1);
+    assertThat(metadata.get(0).get(0)).isEqualTo(emsg);
+  }
+
+  @Test
+  public void decodeMetadata_handlesWrappedMetadata() throws Exception {
+    EventMessage emsg =
+        new EventMessage(
+            EventMessage.ID3_SCHEME_ID,
+            /* value= */ "",
+            /* durationMs= */ 1,
+            /* id= */ 0,
+            encodeTxxxId3Frame("Test description", "Test value"));
+
+    List<Metadata> metadata = runRenderer(eventMessageEncoder.encode(emsg));
+
+    assertThat(metadata).hasSize(1);
+    assertThat(metadata.get(0).length()).isEqualTo(1);
+    TextInformationFrame expectedId3Frame =
+        new TextInformationFrame("TXXX", "Test description", "Test value");
+    assertThat(metadata.get(0).get(0)).isEqualTo(expectedId3Frame);
+  }
+
+  @Test
+  public void decodeMetadata_skipsMalformedWrappedMetadata() throws Exception {
+    EventMessage emsg =
+        new EventMessage(
+            EventMessage.ID3_SCHEME_ID,
+            /* value= */ "",
+            /* durationMs= */ 1,
+            /* id= */ 0,
+            "Not a real ID3 tag".getBytes(ISO_8859_1));
+
+    List<Metadata> metadata = runRenderer(eventMessageEncoder.encode(emsg));
+
+    assertThat(metadata).isEmpty();
+  }
+
+  private static List<Metadata> runRenderer(byte[] input) throws ExoPlaybackException {
+    List<Metadata> metadata = new ArrayList<>();
+    MetadataRenderer renderer = new MetadataRenderer(metadata::add, /* outputLooper= */ null);
+    renderer.replaceStream(
+        new Format[] {EMSG_FORMAT},
+        new FakeSampleStream(EMSG_FORMAT, /* eventDispatcher= */ null, input),
+        /* offsetUs= */ 0L);
+    renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); // Read the format
+    renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); // Read the data
+
+    return Collections.unmodifiableList(metadata);
+  }
+
+  /**
+   * Builds an ID3v2 tag containing a single 'user defined text information frame' (id='TXXX') with
+   * {@code description} and {@code value}.
+   *
+   * <ul>
+   *   <li><a href="http://id3.org/id3v2.4.0-structure">ID3v2 overall structure</a>
+   *   <li><a href="http://id3.org/id3v2.4.0-frames">'user defined text information frame' spec</a>
+   *       (item 4.2.6)
+   * </ul>
+   */
+  private static byte[] encodeTxxxId3Frame(String description, String value) {
+    byte[] id3FrameData =
+        TestUtil.joinByteArrays(
+            "TXXX".getBytes(ISO_8859_1), // ID for a 'user defined text information frame'
+            TestUtil.createByteArray(0, 0, 0, 0), // Frame size (set later)
+            TestUtil.createByteArray(0, 0), // Frame flags
+            TestUtil.createByteArray(0), // Character encoding = ISO-8859-1
+            description.getBytes(ISO_8859_1),
+            TestUtil.createByteArray(0), // String null terminator
+            value.getBytes(ISO_8859_1),
+            TestUtil.createByteArray(0)); // String null terminator
+    int frameSizeIndex = 7;
+    int frameSize = id3FrameData.length - 10;
+    Assertions.checkArgument(
+        frameSize < 128, "frameSize must fit in 7 bits to avoid synch-safe encoding: " + frameSize);
+    id3FrameData[frameSizeIndex] = (byte) frameSize;
+
+    byte[] id3Bytes =
+        TestUtil.joinByteArrays(
+            "ID3".getBytes(ISO_8859_1), // identifier
+            TestUtil.createByteArray(0x04, 0x00), // version
+            TestUtil.createByteArray(0), // Tag flags
+            TestUtil.createByteArray(0, 0, 0, 0), // Tag size (set later)
+            id3FrameData);
+    int tagSizeIndex = 9;
+    int tagSize = id3Bytes.length - 10;
+    Assertions.checkArgument(
+        tagSize < 128, "tagSize must fit in 7 bits to avoid synch-safe encoding: " + tagSize);
+    id3Bytes[tagSizeIndex] = (byte) tagSize;
+    return id3Bytes;
+  }
+}