Read Google Photos motion photo metadata

PiperOrigin-RevId: 338436906

Read Google Photos motion photo metadata
PiperOrigin-RevId: 338436906
175b8eb6 · kimvde · Oliver Woodman · 9bde5d03 · 175b8eb6 · 175b8eb6
Commit 175b8eb6 authored Oct 22, 2020 by kimvde Committed by Oliver Woodman Oct 22, 2020
Showing with 131 additions and 36 deletions
RELEASENOTES.md
library/common/src/main/java/com/google/android/exoplayer2/C.java
library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
testdata/src/test/assets/media/mp4/sample_MP.heic
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -99,6 +99,8 @@
    *   Upgrade IMA SDK dependency to 3.20.1. This brings in a fix for
        companion ads rendering when targeting API 29
        ([#6432](https://github.com/google/ExoPlayer/issues/6432)).
+*   Metadata retriever:
+    *   Parse Google Photos HEIC motion photos metadata.
 ### 2.12.0 (2020-09-11) ###

--- a/library/common/src/main/java/com/google/android/exoplayer2/C.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/C.java
@@ -690,12 +690,14 @@ public final class C {
  public static final int TRACK_TYPE_VIDEO = 2;
  /** A type constant for text tracks. */
  public static final int TRACK_TYPE_TEXT = 3;
+  /** A type constant for image tracks. */
+  public static final int TRACK_TYPE_IMAGE = 4;
  /** A type constant for metadata tracks. */
-  public static final int TRACK_TYPE_METADATA = 4;
+  public static final int TRACK_TYPE_METADATA = 5;
  /** A type constant for camera motion tracks. */
-  public static final int TRACK_TYPE_CAMERA_MOTION = 5;
+  public static final int TRACK_TYPE_CAMERA_MOTION = 6;
  /** A type constant for a fake or empty track. */
-  public static final int TRACK_TYPE_NONE = 6;
+  public static final int TRACK_TYPE_NONE = 7;
  /**
   * Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or
   * equal to this value.

--- a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
@@ -20,21 +20,23 @@ import android.os.Parcel;
 import android.os.Parcelable;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.common.primitives.Longs;
 /** Metadata of a motion photo file. */
 public final class MotionPhoto implements Metadata.Entry {
  /** The start offset of the photo data, in bytes. */
-  public final int photoStartPosition;
+  public final long photoStartPosition;
  /** The size of the photo data, in bytes. */
-  public final int photoSize;
+  public final long photoSize;
  /** The start offset of the video data, in bytes. */
-  public final int videoStartPosition;
+  public final long videoStartPosition;
  /** The size of the video data, in bytes. */
-  public final int videoSize;
+  public final long videoSize;
  /** Creates an instance. */
-  public MotionPhoto(int photoStartPosition, int photoSize, int videoStartPosition, int videoSize) {
+  public MotionPhoto(
+      long photoStartPosition, long photoSize, long videoStartPosition, long videoSize) {
    this.photoStartPosition = photoStartPosition;
    this.photoSize = photoSize;
    this.videoStartPosition = videoStartPosition;
@@ -42,10 +44,10 @@ public final class MotionPhoto implements Metadata.Entry {
  }
  private MotionPhoto(Parcel in) {
-    photoStartPosition = in.readInt();
+    photoStartPosition = in.readLong();
-    photoSize = in.readInt();
+    photoSize = in.readLong();
-    videoStartPosition = in.readInt();
+    videoStartPosition = in.readLong();
-    videoSize = in.readInt();
+    videoSize = in.readLong();
  }
  @Override
@@ -66,10 +68,10 @@ public final class MotionPhoto implements Metadata.Entry {
  @Override
  public int hashCode() {
    int result = 17;
-    result = 31 * result + photoStartPosition;
+    result = 31 * result + Longs.hashCode(photoStartPosition);
-    result = 31 * result + photoSize;
+    result = 31 * result + Longs.hashCode(photoSize);
-    result = 31 * result + videoStartPosition;
+    result = 31 * result + Longs.hashCode(videoStartPosition);
-    result = 31 * result + videoSize;
+    result = 31 * result + Longs.hashCode(videoSize);
    return result;
  }
@@ -89,10 +91,10 @@ public final class MotionPhoto implements Metadata.Entry {
  @Override
  public void writeToParcel(Parcel dest, int flags) {
-    dest.writeInt(photoStartPosition);
+    dest.writeLong(photoStartPosition);
-    dest.writeInt(photoSize);
+    dest.writeLong(photoSize);
-    dest.writeInt(videoStartPosition);
+    dest.writeLong(videoStartPosition);
-    dest.writeInt(videoSize);
+    dest.writeLong(videoSize);
  }
  @Override

--- a/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
@@ -22,6 +22,9 @@ import android.content.Context;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Message;
+import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory;
+import com.google.android.exoplayer2.extractor.ExtractorsFactory;
+import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor;
 import com.google.android.exoplayer2.source.DefaultMediaSourceFactory;
 import com.google.android.exoplayer2.source.MediaPeriod;
 import com.google.android.exoplayer2.source.MediaSource;
@@ -43,8 +46,9 @@ public final class MetadataRetriever {
  /**
   * Retrieves the {@link TrackGroupArray} corresponding to a {@link MediaItem}.
   *
-   * <p>This is equivalent to using {@code retrieveMetadata(new DefaultMediaSourceFactory(context),
+   * <p>This is equivalent to using {@link #retrieveMetadata(MediaSourceFactory, MediaItem)} with a
-   * mediaItem)}.
+   * {@link DefaultMediaSourceFactory} and a {@link DefaultExtractorsFactory} with {@link
+   * Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} set.
   *
   * @param context The {@link Context}.
   * @param mediaItem The {@link MediaItem} whose metadata should be retrieved.
@@ -52,7 +56,12 @@ public final class MetadataRetriever {
   */
  public static ListenableFuture<TrackGroupArray> retrieveMetadata(
      Context context, MediaItem mediaItem) {
-    return retrieveMetadata(new DefaultMediaSourceFactory(context), mediaItem);
+    ExtractorsFactory extractorsFactory =
+        new DefaultExtractorsFactory()
+            .setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA);
+    MediaSourceFactory mediaSourceFactory =
+        new DefaultMediaSourceFactory(context, extractorsFactory);
+    return retrieveMetadata(mediaSourceFactory, mediaItem);
  }
  /**

--- a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
@@ -25,6 +25,7 @@ import android.net.Uri;
 import android.os.SystemClock;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
 import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -37,7 +38,7 @@ import org.junit.runner.RunWith;
 public class MetadataRetrieverTest {
  @Test
-  public void retrieveMetadata_singleMediaItem() throws Exception {
+  public void retrieveMetadata_singleMediaItem_outputsExpectedMetadata() throws Exception {
    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -55,7 +56,7 @@ public class MetadataRetrieverTest {
  }
  @Test
-  public void retrieveMetadata_multipleMediaItems() throws Exception {
+  public void retrieveMetadata_multipleMediaItems_outputsExpectedMetadata() throws Exception {
    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem1 =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -84,7 +85,28 @@ public class MetadataRetrieverTest {
  }
  @Test
-  public void retrieveMetadata_throwsErrorIfCannotLoad() {
+  public void retrieveMetadata_motionPhoto_outputsExpectedMetadata() throws Exception {
+    Context context = ApplicationProvider.getApplicationContext();
+    MediaItem mediaItem =
+        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_MP.heic"));
+    MotionPhoto expectedMotionPhoto =
+        new MotionPhoto(
+            /* photoStartPosition= */ 0,
+            /* photoSize= */ 28_853,
+            /* videoStartPosition= */ 28_869,
+            /* videoSize= */ 28_803);
+    ListenableFuture<TrackGroupArray> trackGroupsFuture = retrieveMetadata(context, mediaItem);
+    TrackGroupArray trackGroups = waitAndGetTrackGroups(trackGroupsFuture);
+    assertThat(trackGroups.length).isEqualTo(1);
+    assertThat(trackGroups.get(0).length).isEqualTo(1);
+    assertThat(trackGroups.get(0).getFormat(0).metadata.length()).isEqualTo(1);
+    assertThat(trackGroups.get(0).getFormat(0).metadata.get(0)).isEqualTo(expectedMotionPhoto);
+  }
+  @Test
+  public void retrieveMetadata_invalidMediaItem_throwsError() {
    Context context = ApplicationProvider.getApplicationContext();
    MediaItem mediaItem =
        MediaItem.fromUri(Uri.parse("asset://android_asset/media/does_not_exist"));

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
@@ -182,6 +182,9 @@ import java.util.List;
  public static final int TYPE_moov = 0x6d6f6f76;
  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_mpvd = 0x6d707664;
+  @SuppressWarnings("ConstantCaseForConstants")
  public static final int TYPE_mvhd = 0x6d766864;
  @SuppressWarnings("ConstantCaseForConstants")

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
@@ -38,6 +38,7 @@ import com.google.android.exoplayer2.extractor.SeekPoint;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
@@ -61,19 +62,27 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new Mp4Extractor()};
  /**
-   * Flags controlling the behavior of the extractor. Possible flag value is {@link
+   * Flags controlling the behavior of the extractor. Possible flag values are {@link
-   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}.
+   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS} and {@link #FLAG_READ_MOTION_PHOTO_METADATA}.
   */
  @Documented
  @Retention(RetentionPolicy.SOURCE)
  @IntDef(
      flag = true,
-      value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS})
+      value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS, FLAG_READ_MOTION_PHOTO_METADATA})
  public @interface Flags {}
  /**
   * Flag to ignore any edit lists in the stream.
   */
  public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1;
+  /**
+   * Flag to extract {@link MotionPhoto} metadata from HEIC motion photos following the Google
+   * Photos Motion Photo File Format V1.1.
+   *
+   * <p>As playback is not supported for motion photos, this flag should only be used for metadata
+   * retrieval use cases.
+   */
+  public static final int FLAG_READ_MOTION_PHOTO_METADATA = 1 << 1;
  /** Parser states. */
  @Documented
@@ -154,7 +163,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  @Override
  public boolean sniff(ExtractorInput input) throws IOException {
-    return Sniffer.sniffUnfragmented(input);
+    return Sniffer.sniffUnfragmented(
+        input, /* acceptHeic= */ (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0);
  }
  @Override
@@ -335,6 +345,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
      this.atomData = atomData;
      parserState = STATE_READING_ATOM_PAYLOAD;
    } else {
+      if (atomType == Atom.TYPE_mpvd && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) {
+        // There is no need to parse the mpvd atom payload. All the necessary information is in the
+        // header.
+        processMpvdBox(
+            /* atomStartPosition= */ input.getPosition() - atomHeaderBytesRead,
+            /* atomHeaderSize= */ atomHeaderBytesRead,
+            atomSize);
+      }
      atomData = null;
      parserState = STATE_READING_ATOM_PAYLOAD;
    }
@@ -663,6 +681,26 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  }
  /**
+   * Processes the Motion Photo Video Data of an HEIC motion photo following the Google Photos
+   * Motion Photo File Format V1.1. This consists in adding a track with the motion photo metadata
+   * and ending playback preparation.
+   */
+  private void processMpvdBox(long atomStartPosition, int atomHeaderSize, long atomSize) {
+    ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput);
+    extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
+    TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
+    MotionPhoto motionPhoto =
+        new MotionPhoto(
+            /* photoStartPosition= */ 0,
+            /* photoSize= */ atomStartPosition,
+            /* videoStartPosition= */ atomStartPosition + atomHeaderSize,
+            /* videoSize= */ atomSize - atomHeaderSize);
+    trackOutput.format(new Format.Builder().setMetadata(new Metadata(motionPhoto)).build());
+    extractorOutput.endTracks();
+  }
+  /**
   * For each sample of each track, calculates accumulated size of all samples which need to be read
   * before this sample can be used.
   */

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
@@ -70,7 +70,7 @@ import java.io.IOException;
   * @throws IOException If an error occurs reading from the input.
   */
  public static boolean sniffFragmented(ExtractorInput input) throws IOException {
-    return sniffInternal(input, true);
+    return sniffInternal(input, /* fragmented= */ true, /* acceptHeic= */ false);
  }
  /**
@@ -82,10 +82,24 @@ import java.io.IOException;
   * @throws IOException If an error occurs reading from the input.
   */
  public static boolean sniffUnfragmented(ExtractorInput input) throws IOException {
-    return sniffInternal(input, false);
+    return sniffInternal(input, /* fragmented= */ false, /* acceptHeic= */ false);
  }
-  private static boolean sniffInternal(ExtractorInput input, boolean fragmented)
+  /**
+   * Returns whether data peeked from the current position in {@code input} is consistent with the
+   * input being an unfragmented MP4 file.
+   *
+   * @param input The extractor input from which to peek data. The peek position will be modified.
+   * @param acceptHeic Whether {@code true} should be returned for HEIC photos.
+   * @return Whether the input appears to be in the unfragmented MP4 format.
+   * @throws IOException If an error occurs reading from the input.
+   */
+  public static boolean sniffUnfragmented(ExtractorInput input, boolean acceptHeic)
+      throws IOException {
+    return sniffInternal(input, /* fragmented= */ false, acceptHeic);
+  }
+  private static boolean sniffInternal(ExtractorInput input, boolean fragmented, boolean acceptHeic)
      throws IOException {
    long inputLength = input.getLength();
    int bytesToSearch = (int) (inputLength == C.LENGTH_UNSET || inputLength > SEARCH_LENGTH
@@ -165,7 +179,7 @@ import java.io.IOException;
          if (i == 1) {
            // This index refers to the minorVersion, not a brand, so skip it.
            buffer.skipBytes(4);
-          } else if (isCompatibleBrand(buffer.readInt())) {
+          } else if (isCompatibleBrand(buffer.readInt(), acceptHeic)) {
            foundGoodFileType = true;
            break;
          }
@@ -185,9 +199,12 @@ import java.io.IOException;
  /**
   * Returns whether {@code brand} is an ftyp atom brand that is compatible with the MP4 extractors.
   */
-  private static boolean isCompatibleBrand(int brand) {
+  private static boolean isCompatibleBrand(int brand, boolean acceptHeic) {
-    // Accept all brands starting '3gp'.
    if (brand >>> 8 == 0x00336770) {
+      // Brand starts with '3gp'.
+      return true;
+    } else if (brand == 0x68656963 && acceptHeic) {
+      // Brand is `heic` and HEIC is supported by the extractor.
      return true;
    }
    for (int compatibleBrand : COMPATIBLE_BRANDS) {

--- a/testdata/src/test/assets/media/mp4/sample_MP.heic
+++ b/testdata/src/test/assets/media/mp4/sample_MP.heic