Merge pull request #10793 from fraunhoferfokus:dash-thumbnail-support

PiperOrigin-RevId: 506261584

Merge pull request #10793 from fraunhoferfokus:dash-thumbnail-support
PiperOrigin-RevId: 506261584
c6569a36 · christosts · f69a2f51 · 40d8f004 · c6569a36 · c6569a36
Commit c6569a36 authored Feb 02, 2023 by christosts
Showing with 143 additions and 13 deletions
RELEASENOTES.md
libraries/common/src/main/java/androidx/media3/common/Format.java
libraries/common/src/test/java/androidx/media3/common/FormatTest.java
libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java
libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
libraries/test_data/src/test/assets/media/mpd/sample_mpd_images
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -37,6 +37,9 @@
        `Subtitle.getEventTime` if a subtitle file contains no cues.
    *   SubRip: Add support for UTF-16 files if they start with a byte order
        mark.
+*   DASH:
+    *   Add full parsing for image adaptation sets, including tile counts
+        ([#3752](https://github.com/google/ExoPlayer/issues/3752)).
 *   UI:
    *   Fix the deprecated
        `PlayerView.setControllerVisibilityListener(PlayerControlView.VisibilityListener)`

--- a/libraries/common/src/main/java/androidx/media3/common/Format.java
+++ b/libraries/common/src/main/java/androidx/media3/common/Format.java
@@ -105,6 +105,13 @@ import java.util.UUID;
 * <ul>
 *   <li>{@link #accessibilityChannel}
 * </ul>
+ *
+ * <h2 id="image-formats">Fields relevant to image formats</h2>
+ *
+ * <ul>
+ *   <li>{@link #tileCountHorizontal}
+ *   <li>{@link #tileCountVertical}
+ * </ul>
 */
 public final class Format implements Bundleable {
@@ -165,6 +172,11 @@ public final class Format implements Bundleable {
    private int accessibilityChannel;
+    // Image specific
+    private int tileCountHorizontal;
+    private int tileCountVertical;
    // Provided by the source.
    private @C.CryptoType int cryptoType;
@@ -188,6 +200,9 @@ public final class Format implements Bundleable {
      pcmEncoding = NO_VALUE;
      // Text specific.
      accessibilityChannel = NO_VALUE;
+      // Image specific.
+      tileCountHorizontal = NO_VALUE;
+      tileCountVertical = NO_VALUE;
      // Provided by the source.
      cryptoType = C.CRYPTO_TYPE_NONE;
    }
@@ -232,6 +247,9 @@ public final class Format implements Bundleable {
      this.encoderPadding = format.encoderPadding;
      // Text specific.
      this.accessibilityChannel = format.accessibilityChannel;
+      // Image specific.
+      this.tileCountHorizontal = format.tileCountHorizontal;
+      this.tileCountVertical = format.tileCountVertical;
      // Provided by the source.
      this.cryptoType = format.cryptoType;
    }
@@ -607,6 +625,32 @@ public final class Format implements Bundleable {
      return this;
    }
+    // Image specific.
+    /**
+     * Sets {@link Format#tileCountHorizontal}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountHorizontal The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setTileCountHorizontal(int tileCountHorizontal) {
+      this.tileCountHorizontal = tileCountHorizontal;
+      return this;
+    }
+    /**
+     * Sets {@link Format#tileCountVertical}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountVertical The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setTileCountVertical(int tileCountVertical) {
+      this.tileCountVertical = tileCountVertical;
+      return this;
+    }
    // Provided by source.
    /**
@@ -779,6 +823,15 @@ public final class Format implements Bundleable {
  /** The Accessibility channel, or {@link #NO_VALUE} if not known or applicable. */
  @UnstableApi public final int accessibilityChannel;
+  // Image specific.
+  /**
+   * The number of horizontal tiles in an image, or {@link #NO_VALUE} if not known or applicable.
+   */
+  @UnstableApi public final int tileCountHorizontal;
+  /** The number of vertical tiles in an image, or {@link #NO_VALUE} if not known or applicable. */
+  @UnstableApi public final int tileCountVertical;
  // Provided by source.
  /**
@@ -1008,6 +1061,9 @@ public final class Format implements Bundleable {
    encoderPadding = builder.encoderPadding == NO_VALUE ? 0 : builder.encoderPadding;
    // Text specific.
    accessibilityChannel = builder.accessibilityChannel;
+    // Image specific.
+    tileCountHorizontal = builder.tileCountHorizontal;
+    tileCountVertical = builder.tileCountVertical;
    // Provided by source.
    if (builder.cryptoType == C.CRYPTO_TYPE_NONE && drmInitData != null) {
      // Encrypted content cannot use CRYPTO_TYPE_NONE.
@@ -1268,6 +1324,9 @@ public final class Format implements Bundleable {
      result = 31 * result + encoderPadding;
      // Text specific.
      result = 31 * result + accessibilityChannel;
+      // Image specific.
+      result = 31 * result + tileCountHorizontal;
+      result = 31 * result + tileCountVertical;
      // Provided by the source.
      result = 31 * result + cryptoType;
      hashCode = result;
@@ -1304,6 +1363,8 @@ public final class Format implements Bundleable {
        && encoderDelay == other.encoderDelay
        && encoderPadding == other.encoderPadding
        && accessibilityChannel == other.accessibilityChannel
+        && tileCountHorizontal == other.tileCountHorizontal
+        && tileCountVertical == other.tileCountVertical
        && cryptoType == other.cryptoType
        && Float.compare(frameRate, other.frameRate) == 0
        && Float.compare(pixelWidthHeightRatio, other.pixelWidthHeightRatio) == 0
@@ -1500,6 +1561,8 @@ public final class Format implements Bundleable {
  private static final String FIELD_ENCODER_PADDING = Util.intToStringMaxRadix(27);
  private static final String FIELD_ACCESSIBILITY_CHANNEL = Util.intToStringMaxRadix(28);
  private static final String FIELD_CRYPTO_TYPE = Util.intToStringMaxRadix(29);
+  private static final String FIELD_TILE_COUNT_HORIZONTAL = Util.intToStringMaxRadix(30);
+  private static final String FIELD_TILE_COUNT_VERTICAL = Util.intToStringMaxRadix(31);
  @UnstableApi
  @Override
@@ -1557,6 +1620,9 @@ public final class Format implements Bundleable {
    bundle.putInt(FIELD_ENCODER_PADDING, encoderPadding);
    // Text specific.
    bundle.putInt(FIELD_ACCESSIBILITY_CHANNEL, accessibilityChannel);
+    // Image specific.
+    bundle.putInt(FIELD_TILE_COUNT_HORIZONTAL, tileCountHorizontal);
+    bundle.putInt(FIELD_TILE_COUNT_VERTICAL, tileCountVertical);
    // Source specific.
    bundle.putInt(FIELD_CRYPTO_TYPE, cryptoType);
    return bundle;
@@ -1621,6 +1687,10 @@ public final class Format implements Bundleable {
        // Text specific.
        .setAccessibilityChannel(
            bundle.getInt(FIELD_ACCESSIBILITY_CHANNEL, DEFAULT.accessibilityChannel))
+        // Image specific.
+        .setTileCountHorizontal(
+            bundle.getInt(FIELD_TILE_COUNT_HORIZONTAL, DEFAULT.tileCountHorizontal))
+        .setTileCountVertical(bundle.getInt(FIELD_TILE_COUNT_VERTICAL, DEFAULT.tileCountVertical))
        // Source specific.
        .setCryptoType(bundle.getInt(FIELD_CRYPTO_TYPE, DEFAULT.cryptoType));

--- a/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
+++ b/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
@@ -111,6 +111,8 @@ public final class FormatTest {
        .setEncoderPadding(1002)
        .setAccessibilityChannel(2)
        .setCryptoType(C.CRYPTO_TYPE_CUSTOM_BASE)
+        .setTileCountHorizontal(20)
+        .setTileCountVertical(40)
        .build();
  }

--- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java
+++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java
@@ -1058,9 +1058,11 @@ public final class DashMediaSource extends BaseMediaSource {
    for (int i = 0; i < period.adaptationSets.size(); i++) {
      AdaptationSet adaptationSet = period.adaptationSets.get(i);
      List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude other adaptation sets from duration calculations, if we have at least one audio or
-      // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
+      // video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029.
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      boolean adaptationSetIsNotAudioVideo =
+          adaptationSet.type != C.TRACK_TYPE_AUDIO && adaptationSet.type != C.TRACK_TYPE_VIDEO;
+      if ((haveAudioVideoAdaptationSets && adaptationSetIsNotAudioVideo)
          || representations.isEmpty()) {
        continue;
      }
@@ -1090,9 +1092,11 @@ public final class DashMediaSource extends BaseMediaSource {
    for (int i = 0; i < period.adaptationSets.size(); i++) {
      AdaptationSet adaptationSet = period.adaptationSets.get(i);
      List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude other adaptation sets from duration calculations, if we have at least one audio or
-      // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
+      // video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      boolean adaptationSetIsNotAudioVideo =
+          adaptationSet.type != C.TRACK_TYPE_AUDIO && adaptationSet.type != C.TRACK_TYPE_VIDEO;
+      if ((haveAudioVideoAdaptationSets && adaptationSetIsNotAudioVideo)
          || representations.isEmpty()) {
        continue;
      }

--- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
+++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java
@@ -557,6 +557,8 @@ public class DashManifestParser extends DefaultHandler
                ? C.TRACK_TYPE_VIDEO
                : MimeTypes.BASE_TYPE_TEXT.equals(contentType)
                    ? C.TRACK_TYPE_TEXT
+                    : MimeTypes.BASE_TYPE_IMAGE.equals(contentType)
+                        ? C.TRACK_TYPE_IMAGE
                        : C.TRACK_TYPE_UNKNOWN;
  }
@@ -810,6 +812,7 @@ public class DashManifestParser extends DefaultHandler
    roleFlags |= parseRoleFlagsFromAccessibilityDescriptors(accessibilityDescriptors);
    roleFlags |= parseRoleFlagsFromProperties(essentialProperties);
    roleFlags |= parseRoleFlagsFromProperties(supplementalProperties);
+    @Nullable Pair<Integer, Integer> tileCounts = parseTileCountFromProperties(essentialProperties);
    Format.Builder formatBuilder =
        new Format.Builder()
@@ -820,7 +823,9 @@ public class DashManifestParser extends DefaultHandler
            .setPeakBitrate(bitrate)
            .setSelectionFlags(selectionFlags)
            .setRoleFlags(roleFlags)
-            .setLanguage(language);
+            .setLanguage(language)
+            .setTileCountHorizontal(tileCounts != null ? tileCounts.first : Format.NO_VALUE)
+            .setTileCountVertical(tileCounts != null ? tileCounts.second : Format.NO_VALUE);
    if (MimeTypes.isVideo(sampleMimeType)) {
      formatBuilder.setWidth(width).setHeight(height).setFrameRate(frameRate);
@@ -1629,6 +1634,41 @@ public class DashManifestParser extends DefaultHandler
    return attributeValue.split(",");
  }
+  // Thumbnail tile information parsing
+  /**
+   * Parses given descriptors for thumbnail tile information.
+   *
+   * @param essentialProperties List of descriptors that contain thumbnail tile information.
+   * @return A pair of Integer values, where the first is the count of horizontal tiles and the
+   *     second is the count of vertical tiles, or null if no thumbnail tile information is found.
+   */
+  @Nullable
+  protected Pair<Integer, Integer> parseTileCountFromProperties(
+      List<Descriptor> essentialProperties) {
+    for (int i = 0; i < essentialProperties.size(); i++) {
+      Descriptor descriptor = essentialProperties.get(i);
+      if ((Ascii.equalsIgnoreCase("http://dashif.org/thumbnail_tile", descriptor.schemeIdUri)
+              || Ascii.equalsIgnoreCase(
+                  "http://dashif.org/guidelines/thumbnail_tile", descriptor.schemeIdUri))
+          && descriptor.value != null) {
+        String size = descriptor.value;
+        String[] sizeSplit = Util.split(size, "x");
+        if (sizeSplit.length != 2) {
+          continue;
+        }
+        try {
+          int tileCountHorizontal = Integer.parseInt(sizeSplit[0]);
+          int tileCountVertical = Integer.parseInt(sizeSplit[1]);
+          return Pair.create(tileCountHorizontal, tileCountVertical);
+        } catch (NumberFormatException e) {
+          // Ignore property if it's malformed.
+        }
+      }
+    }
+    return null;
+  }
  // Utility methods.
  /**

--- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
+++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java
@@ -252,11 +252,19 @@ public class DashManifestParserTest {
                ApplicationProvider.getApplicationContext(), SAMPLE_MPD_IMAGES));
    AdaptationSet adaptationSet = manifest.getPeriod(0).adaptationSets.get(0);
-    Format format = adaptationSet.representations.get(0).format;
+    Format format0 = adaptationSet.representations.get(0).format;
+    Format format1 = adaptationSet.representations.get(1).format;
-    assertThat(format.sampleMimeType).isEqualTo("image/jpeg");
-    assertThat(format.width).isEqualTo(320);
+    assertThat(format0.sampleMimeType).isEqualTo("image/jpeg");
-    assertThat(format.height).isEqualTo(180);
+    assertThat(format0.width).isEqualTo(320);
+    assertThat(format0.height).isEqualTo(180);
+    assertThat(format0.tileCountHorizontal).isEqualTo(12);
+    assertThat(format0.tileCountVertical).isEqualTo(16);
+    assertThat(format1.sampleMimeType).isEqualTo("image/jpeg");
+    assertThat(format1.width).isEqualTo(640);
+    assertThat(format1.height).isEqualTo(360);
+    assertThat(format1.tileCountHorizontal).isEqualTo(2);
+    assertThat(format1.tileCountVertical).isEqualTo(4);
  }
  @Test

--- a/libraries/test_data/src/test/assets/media/mpd/sample_mpd_images
+++ b/libraries/test_data/src/test/assets/media/mpd/sample_mpd_images
@@ -4,7 +4,10 @@
  <AdaptationSet id="3" mimeType="image/jpeg" contentType="image">
    <SegmentTemplate media="$RepresentationID$/tile_$Number$.jpg" duration="100" startNumber="1"/>
    <Representation bandwidth="1234" id="images_320x180" width="320" height="180">
-      <EssentialProperty schemeIdUri="http://dashif.org/thumbnail_tile" value="title"/>
+      <EssentialProperty schemeIdUri="http://dashif.org/thumbnail_tile" value="12x16"/>
+    </Representation>
+    <Representation bandwidth="2345" id="images_640x360" width="640" height="360">
+      <EssentialProperty schemeIdUri="http://dashif.org/guidelines/thumbnail_tile" value="2x4"/>
    </Representation>
  </AdaptationSet>
 </Period>