Avoid throwing for still photo metadata retrieval

PiperOrigin-RevId: 338497163

Avoid throwing for still photo metadata retrieval
PiperOrigin-RevId: 338497163
521a2207 · kimvde · Oliver Woodman · 11918204 · 521a2207 · 521a2207
Commit 521a2207 authored Oct 22, 2020 by kimvde Committed by Oliver Woodman Oct 23, 2020
Showing with 92 additions and 43 deletions
library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
testdata/src/test/assets/media/mp4/sample_still_photo.heic
--- a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
@@ -30,6 +30,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.util.concurrent.ListenableFuture;
 import java.util.concurrent.ExecutionException;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -37,9 +38,15 @@ import org.junit.runner.RunWith;
 @RunWith(AndroidJUnit4.class)
 public class MetadataRetrieverTest {
+  private Context context;
+  @Before
+  public void setUp() throws Exception {
+    context = ApplicationProvider.getApplicationContext();
+  }
  @Test
  public void retrieveMetadata_singleMediaItem_outputsExpectedMetadata() throws Exception {
-    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -57,7 +64,6 @@ public class MetadataRetrieverTest {
  @Test
  public void retrieveMetadata_multipleMediaItems_outputsExpectedMetadata() throws Exception {
-    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem1 =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
    MediaItem mediaItem2 =
@@ -85,8 +91,7 @@ public class MetadataRetrieverTest {
  }
  @Test
-  public void retrieveMetadata_motionPhoto_outputsExpectedMetadata() throws Exception {
+  public void retrieveMetadata_heicMotionPhoto_outputsExpectedMetadata() throws Exception {
-    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_MP.heic"));
    MotionPhoto expectedMotionPhoto =
@@ -106,8 +111,20 @@ public class MetadataRetrieverTest {
  }
  @Test
+  public void retrieveMetadata_heicStillPhoto_outputsEmptyMetadata() throws Exception {
+    MediaItem mediaItem =
+        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_still_photo.heic"));
+    ListenableFuture<TrackGroupArray> trackGroupsFuture = retrieveMetadata(context, mediaItem);
+    TrackGroupArray trackGroups = waitAndGetTrackGroups(trackGroupsFuture);
+    assertThat(trackGroups.length).isEqualTo(1);
+    assertThat(trackGroups.get(0).length).isEqualTo(1);
+    assertThat(trackGroups.get(0).getFormat(0).metadata).isNull();
+  }
+  @Test
  public void retrieveMetadata_invalidMediaItem_throwsError() {
-    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/does_not_exist"));

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
@@ -16,6 +16,8 @@
 package com.google.android.exoplayer2.extractor.mp4;
 import static com.google.android.exoplayer2.extractor.mp4.AtomParsers.parseTraks;
+import static com.google.android.exoplayer2.extractor.mp4.Sniffer.BRAND_HEIC;
+import static com.google.android.exoplayer2.extractor.mp4.Sniffer.BRAND_QUICKTIME;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Util.castNonNull;
 import static java.lang.Math.max;
@@ -94,8 +96,15 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  private static final int STATE_READING_ATOM_PAYLOAD = 1;
  private static final int STATE_READING_SAMPLE = 2;
-  /** Brand stored in the ftyp atom for QuickTime media. */
+  /** Supported file types. */
-  private static final int BRAND_QUICKTIME = 0x71742020;
+  @Documented
+  @Retention(RetentionPolicy.SOURCE)
+  @IntDef({FILE_TYPE_MP4, FILE_TYPE_QUICKTIME, FILE_TYPE_HEIC})
+  private @interface FileType {}
+  private static final int FILE_TYPE_MP4 = 0;
+  private static final int FILE_TYPE_QUICKTIME = 1;
+  private static final int FILE_TYPE_HEIC = 2;
  /**
   * When seeking within the source, if the offset is greater than or equal to this value (or the
@@ -133,10 +142,12 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  // Extractor outputs.
  private @MonotonicNonNull ExtractorOutput extractorOutput;
  private Mp4Track @MonotonicNonNull [] tracks;
  private long @MonotonicNonNull [][] accumulatedSampleSizes;
  private int firstVideoTrackIndex;
  private long durationUs;
-  private boolean isQuickTime;
+  @FileType private int fileType;
+  @Nullable private MotionPhoto motionPhoto;
  /**
   * Creates a new extractor for unfragmented MP4 streams.
@@ -290,6 +301,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    if (atomHeaderBytesRead == 0) {
      // Read the standard length atom header.
      if (!input.readFully(atomHeader.getData(), 0, Atom.HEADER_SIZE, true)) {
+        processEndOfStreamReadingAtomHeader();
        return false;
      }
      atomHeaderBytesRead = Atom.HEADER_SIZE;
@@ -345,14 +357,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
      this.atomData = atomData;
      parserState = STATE_READING_ATOM_PAYLOAD;
    } else {
-      if (atomType == Atom.TYPE_mpvd && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) {
+      processUnparsedAtom(input.getPosition() - atomHeaderBytesRead);
-        // There is no need to parse the mpvd atom payload. All the necessary information is in the
-        // header.
-        processMpvdBox(
-            /* atomStartPosition= */ input.getPosition() - atomHeaderBytesRead,
-            /* atomHeaderSize= */ atomHeaderBytesRead,
-            atomSize);
-      }
      atomData = null;
      parserState = STATE_READING_ATOM_PAYLOAD;
    }
@@ -374,7 +379,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    if (atomData != null) {
      input.readFully(atomData.getData(), atomHeaderBytesRead, (int) atomPayloadSize);
      if (atomType == Atom.TYPE_ftyp) {
-        isQuickTime = processFtypAtom(atomData);
+        fileType = processFtypAtom(atomData);
      } else if (!containerAtoms.isEmpty()) {
        containerAtoms.peek().add(new Atom.LeafAtom(atomType, atomData));
      }
@@ -418,6 +423,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    // Process metadata.
    @Nullable Metadata udtaMetadata = null;
+    boolean isQuickTime = fileType == FILE_TYPE_QUICKTIME;
    GaplessInfoHolder gaplessInfoHolder = new GaplessInfoHolder();
    @Nullable Atom.LeafAtom udta = moov.getLeafAtomOfType(Atom.TYPE_udta);
    if (udta != null) {
@@ -655,6 +661,19 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    }
  }
+  /** Processes the end of stream in case there is not atom left to read. */
+  private void processEndOfStreamReadingAtomHeader() {
+    if (fileType == FILE_TYPE_HEIC && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) {
+      // Add image track and prepare media.
+      ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput);
+      TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
+      @Nullable Metadata metadata = motionPhoto == null ? null : new Metadata(motionPhoto);
+      trackOutput.format(new Format.Builder().setMetadata(metadata).build());
+      extractorOutput.endTracks();
+      extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
+    }
+  }
  /**
   * Possibly skips the version and flags fields (1+3 byte) of a full meta atom of the {@code
   * input}.
@@ -680,24 +699,18 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    }
  }
-  /**
+  /** Processes an atom whose payload does not need to be parsed. */
-   * Processes the Motion Photo Video Data of an HEIC motion photo following the Google Photos
+  private void processUnparsedAtom(long atomStartPosition) {
-   * Motion Photo File Format V1.1. This consists in adding a track with the motion photo metadata
+    if (atomType == Atom.TYPE_mpvd) {
-   * and ending playback preparation.
+      // The input is an HEIC motion photo following the Google Photos Motion Photo File Format
-   */
+      // V1.1.
-  private void processMpvdBox(long atomStartPosition, int atomHeaderSize, long atomSize) {
+      motionPhoto =
-    ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput);
-    extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
-    TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
-    MotionPhoto motionPhoto =
          new MotionPhoto(
              /* photoStartPosition= */ 0,
              /* photoSize= */ atomStartPosition,
-            /* videoStartPosition= */ atomStartPosition + atomHeaderSize,
+              /* videoStartPosition= */ atomStartPosition + atomHeaderBytesRead,
-            /* videoSize= */ atomSize - atomHeaderSize);
+              /* videoSize= */ atomSize - atomHeaderBytesRead);
-    trackOutput.format(new Format.Builder().setMetadata(new Metadata(motionPhoto)).build());
+    }
-    extractorOutput.endTracks();
  }
  /**
@@ -779,24 +792,39 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  }
  /**
-   * Process an ftyp atom to determine whether the media is QuickTime.
+   * Process an ftyp atom to determine the corresponding {@link FileType}.
   *
   * @param atomData The ftyp atom data.
-   * @return Whether the media is QuickTime.
+   * @return The {@link FileType}.
   */
-  private static boolean processFtypAtom(ParsableByteArray atomData) {
+  @FileType
+  private static int processFtypAtom(ParsableByteArray atomData) {
    atomData.setPosition(Atom.HEADER_SIZE);
    int majorBrand = atomData.readInt();
-    if (majorBrand == BRAND_QUICKTIME) {
+    @FileType int fileType = brandToFileType(majorBrand);
-      return true;
+    if (fileType != FILE_TYPE_MP4) {
+      return fileType;
    }
    atomData.skipBytes(4); // minor_version
    while (atomData.bytesLeft() > 0) {
-      if (atomData.readInt() == BRAND_QUICKTIME) {
+      fileType = brandToFileType(atomData.readInt());
-        return true;
+      if (fileType != FILE_TYPE_MP4) {
+        return fileType;
      }
    }
-    return false;
+    return FILE_TYPE_MP4;
+  }
+  @FileType
+  private static int brandToFileType(int brand) {
+    switch (brand) {
+      case BRAND_QUICKTIME:
+        return FILE_TYPE_QUICKTIME;
+      case BRAND_HEIC:
+        return FILE_TYPE_HEIC;
+      default:
+        return FILE_TYPE_MP4;
+    }
  }
  /** Returns whether the extractor should decode a leaf atom with type {@code atom}. */

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
@@ -26,6 +26,11 @@ import java.io.IOException;
 */
 /* package */ final class Sniffer {
+  /** Brand stored in the ftyp atom for QuickTime media. */
+  public static final int BRAND_QUICKTIME = 0x71742020;
+  /** Brand stored in the ftyp atom for HEIC media. */
+  public static final int BRAND_HEIC = 0x68656963;
  /** The maximum number of bytes to peek when sniffing. */
  private static final int SEARCH_LENGTH = 4 * 1024;
@@ -54,7 +59,7 @@ import java.io.IOException;
        0x66347620, // f4v[space]
        0x6b646469, // kddi
        0x4d345650, // M4VP
-        0x71742020, // qt[space][space], Apple QuickTime
+        BRAND_QUICKTIME, // qt[space][space]
        0x4d534e56, // MSNV, Sony PSP
        0x64627931, // dby1, Dolby Vision
        0x69736d6c, // isml
@@ -203,8 +208,7 @@ import java.io.IOException;
    if (brand >>> 8 == 0x00336770) {
      // Brand starts with '3gp'.
      return true;
-    } else if (brand == 0x68656963 && acceptHeic) {
+    } else if (brand == BRAND_HEIC && acceptHeic) {
-      // Brand is `heic` and HEIC is supported by the extractor.
      return true;
    }
    for (int compatibleBrand : COMPATIBLE_BRANDS) {

--- a/testdata/src/test/assets/media/mp4/sample_still_photo.heic
+++ b/testdata/src/test/assets/media/mp4/sample_still_photo.heic