DashManifest has been enhanced to allow fetching Thumbnail Meta Data for a video…

DashManifest has been enhanced to allow fetching Thumbnail Meta Data for a video position and with that allowing to display thumbnail images while seeking through a video.
The demo app has beed enhanced to use this new API and display thumbnail images above the seek bar while scrubbing.

demos/main/src/main/assets/media.exolist.json

@@ -632,6 +632,23 @@
     ]
   },
   {
+    "name": "Thumbnails",
+    "samples": [
+      {
+        "name": "Single adaptation set, 7 tiles at 10x1, each thumb 320x180",
+        "uri": "https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_tiled_thumbnails.mpd"
+      },
+      {
+        "name": "Single adaptation set, SegmentTemplate with SegmentTimeline",
+        "uri": "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel-tiled-thumbnails-timeline.ism/.mpd"
+      },
+      {
+        "name": "Single adaptation set, SegmentTemplate with SegmentNumber",
+        "uri": "https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel-tiled-thumbnails-numbered.ism/.mpd"
+      }
+    ]
+  },
+  {
     "name": "Misc",
     "samples": [
       {

demos/main/src/main/java/com/google/android/exoplayer2/demo/PlayerActivity.java

@@ -75,6 +75,7 @@ public class PlayerActivity extends AppCompatActivity
   protected LinearLayout debugRootView;
   protected TextView debugTextView;
   protected @Nullable ExoPlayer player;
+  DefaultThumbnailTimeBar timeBar;
 
   private boolean isShowingTrackSelectionDialog;
   private Button selectTracksButton;
@@ -117,6 +118,8 @@ public class PlayerActivity extends AppCompatActivity
     playerView.setErrorMessageProvider(new PlayerErrorMessageProvider());
     playerView.requestFocus();
 
+    timeBar = playerView.findViewById(R.id.exo_progress);
+
     if (savedInstanceState != null) {
       trackSelectionParameters =
           TrackSelectionParameters.fromBundle(
@@ -282,6 +285,7 @@ public class PlayerActivity extends AppCompatActivity
       player.setPlayWhenReady(startAutoPlay);
       playerView.setPlayer(player);
       configurePlayerWithServerSideAdsLoader();
+      timeBar.setThumbnailUtils(new DefaultThumbnailProvider(player, timeBar));
       debugViewHelper = new DebugTextViewHelper(player, debugTextView);
       debugViewHelper.start();
     }

library/core/src/main/java/com/google/android/exoplayer2/thumbnail/ThumbnailDescription.java

+package com.google.android.exoplayer2.thumbnail;
+
+import android.net.Uri;
+
+public class ThumbnailDescription {
+
+  private final String id;
+  private final Uri uri;
+  private final int bitrate;
+  private final int rows;
+  private final int columns;
+  private final long startTimeMs;
+  private final long durationMs;
+  private final int imageWidth;            // Image width (Pixel)
+  private final int imageHeight;           // Image height (Pixel)
+
+  public ThumbnailDescription(String id, Uri uri, int bitrate, int rows, int columns, long startTimeMs, long durationMs, int imageWidth, int imageHeight) {
+    this.id = id;
+    this.uri = uri;
+    this.bitrate = bitrate;
+    this.rows = rows;
+    this.columns = columns;
+    this.startTimeMs = startTimeMs;
+    this.durationMs = durationMs;
+    this.imageWidth = imageWidth;
+    this.imageHeight = imageHeight;
+  }
+
+  public Uri getUri() {
+    return uri;
+  }
+
+  public int getBitrate() {
+    return bitrate;
+  }
+
+  public int getRows() {
+    return rows;
+  }
+
+  public int getColumns() {
+    return columns;
+  }
+
+  public long getStartTimeMs() {
+    return startTimeMs;
+  }
+
+  public long getDurationMs() {
+    return durationMs;
+  }
+
+  public int getImageWidth() {
+    return imageWidth;
+  }
+
+  public int getImageHeight() {
+    return imageHeight;
+  }
+}

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifest.java

@@ -15,12 +15,20 @@
  */
 package com.google.android.exoplayer2.source.dash.manifest;
 
+import static com.google.android.exoplayer2.util.Util.castNonNull;
+
 import android.net.Uri;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.offline.FilterableManifest;
 import com.google.android.exoplayer2.offline.StreamKey;
+import com.google.android.exoplayer2.source.dash.BaseUrlExclusionList;
+import com.google.android.exoplayer2.source.dash.DashSegmentIndex;
+import com.google.android.exoplayer2.source.dash.DashUtil;
+import com.google.android.exoplayer2.thumbnail.ThumbnailDescription;
+import com.google.android.exoplayer2.upstream.DataSpec;
 import com.google.android.exoplayer2.util.Util;
+import com.google.common.base.Ascii;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -136,6 +144,90 @@ public class DashManifest implements FilterableManifest<DashManifest> {
     return Util.msToUs(getPeriodDurationMs(index));
   }
 
+  /**
+   * Returns a List of ThumbnailDescription for a given periodPosition,
+   * or null if no AdaptionSet of type C.TRACK_TYPE_IMAGE is available.
+   * @param periodPositionMs the period position to get ThumbnailDescription for, e.g. current player position.
+   * @return List of ThumbnailDescription from all Representations, or null if Thumbnails are not available in the DashManifest.
+   */
+  @Nullable
+  public List<ThumbnailDescription> getThumbnailDescriptions(long periodPositionMs) {
+    ArrayList<ThumbnailDescription> thumbnailDescriptions = new ArrayList<>();
+
+    long periodPositionUs = Util.msToUs(periodPositionMs);
+    BaseUrlExclusionList baseUrlExclusionList = new BaseUrlExclusionList();
+
+    boolean isTrackTypeImageAvailable = false;
+    for (int i = 0; i < getPeriodCount(); i++) {
+      Period period = getPeriod(i);
+      long periodStartUs = Util.msToUs(period.startMs);
+      long periodDurationUs = getPeriodDurationUs(i);
+
+      List<AdaptationSet> adaptationSets = period.adaptationSets;
+      for (int j = 0; j < adaptationSets.size(); j++) {
+        AdaptationSet adaptationSet = adaptationSets.get(j);
+        if (adaptationSet.type != C.TRACK_TYPE_IMAGE) {
+          continue;
+        }
+        isTrackTypeImageAvailable = true;
+
+        // thumbnails found
+        List<Representation> representations = adaptationSet.representations;
+        for (int k = 0; k < representations.size(); k++) {
+
+          Representation representation = representations.get(k);
+          DashSegmentIndex index = representation.getIndex();
+          if (index == null) {
+            continue;
+          }
+
+          String id = representation.format.id;
+          int bitrate = representation.format.bitrate;
+          int imageWidth = representation.format.width;
+          int imageHeight = representation.format.height;
+
+          // get size XxY, e.g. 10x20, where 10 is column count and 20 is row count
+          int rows = 1;
+          int cols = 1;
+          for (int m = 0; m < representation.essentialProperties.size(); m++) {
+            Descriptor descriptor = representation.essentialProperties.get(m);
+            if ((Ascii.equalsIgnoreCase("http://dashif.org/thumbnail_tile", descriptor.schemeIdUri) || Ascii.equalsIgnoreCase("http://dashif.org/guidelines/thumbnail_tile", descriptor.schemeIdUri)) && descriptor.value != null) {
+              String size = descriptor.value;
+              String[] sizeSplit = size.split("x");
+              if (sizeSplit.length != 2) {
+                continue;
+              }
+              cols = Integer.parseInt(sizeSplit[0]);
+              rows = Integer.parseInt(sizeSplit[1]);
+            }
+          }
+
+          long now = Util.getNowUnixTimeMs(C.TIME_UNSET);
+          String baseUrl = castNonNull(baseUrlExclusionList.selectBaseUrl(representation.baseUrls)).url;
+
+          // calculate the correct positionUs, which is FirstAvailableSegment.time + playerPosition, use that to get the correct segment
+          long firstSegmentNum = index.getFirstAvailableSegmentNum(periodDurationUs, Util.msToUs(now));
+          long firstStartTimeUs = index.getTimeUs(firstSegmentNum);
+          long positionUs = firstStartTimeUs + periodPositionUs;
+          long segmentNumber = index.getSegmentNum(positionUs, periodDurationUs);
+
+          long segmentStartTimeUs = periodStartUs + index.getTimeUs(segmentNumber);
+          long segmentDurationUs = index.getDurationUs(segmentNumber, periodDurationUs);
+
+          RangedUri rangedUri = index.getSegmentUrl(segmentNumber);
+          DataSpec dataSpec = DashUtil.buildDataSpec(representation, baseUrl, rangedUri, /* flags= */ 0);
+          Uri uri = dataSpec.uri;
+          ThumbnailDescription thumbnailDescription = new ThumbnailDescription(id, uri, bitrate, rows, cols, Util.usToMs(segmentStartTimeUs - (dynamic ? firstStartTimeUs : 0)), Util.usToMs(segmentDurationUs), imageWidth, imageHeight);
+          thumbnailDescriptions.add(thumbnailDescription);
+        }
+      }
+    }
+    if (isTrackTypeImageAvailable) {
+      return thumbnailDescriptions;
+    }
+    return null;
+  }
+
   @Override
   public final DashManifest copy(List<StreamKey> streamKeys) {
     LinkedList<StreamKey> keys = new LinkedList<>(streamKeys);

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java

@@ -555,7 +555,9 @@ public class DashManifestParser extends DefaultHandler
                 ? C.TRACK_TYPE_VIDEO
                 : MimeTypes.BASE_TYPE_TEXT.equals(contentType)
                     ? C.TRACK_TYPE_TEXT
-                    : C.TRACK_TYPE_UNKNOWN;
+                    : MimeTypes.BASE_TYPE_IMAGE.equals(contentType)
+                        ? C.TRACK_TYPE_IMAGE
+                        : C.TRACK_TYPE_UNKNOWN;
   }
 
   /**