Add support for playing JPEG motion photos

PiperOrigin-RevId: 352413375

RELEASENOTES.md

@@ -147,6 +147,8 @@
         ([#8393](https://github.com/google/ExoPlayer/issues/8393)).
     *   Handle sample size mismatches between raw audio `stsd` information and
         `stsz` fixed sample size in MP4 extractors.
+    *   Add support for playing JPEG motion photos
+        ([#5405](https://github.com/google/ExoPlayer/issues/5405)).
 *   Track selection:
     *   Allow parallel adaptation for video and audio
         ([#5111](https://github.com/google/ExoPlayer/issues/5111)).

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/jpeg/JpegExtractor.java

@@ -48,6 +48,7 @@ public final class JpegExtractor implements Extractor {
     STATE_READING_SEGMENT_LENGTH,
     STATE_READING_SEGMENT,
     STATE_SNIFFING_MOTION_PHOTO_VIDEO,
+    STATE_READING_MOTION_PHOTO_VIDEO,
     STATE_ENDED,
   })
   private @interface State {}
@@ -56,7 +57,8 @@ public final class JpegExtractor implements Extractor {
   private static final int STATE_READING_SEGMENT_LENGTH = 1;
   private static final int STATE_READING_SEGMENT = 2;
   private static final int STATE_SNIFFING_MOTION_PHOTO_VIDEO = 4;
-  private static final int STATE_ENDED = 5;
+  private static final int STATE_READING_MOTION_PHOTO_VIDEO = 5;
+  private static final int STATE_ENDED = 6;
 
   private static final int JPEG_EXIF_HEADER_LENGTH = 12;
   private static final long EXIF_HEADER = 0x45786966; // Exif
@@ -65,6 +67,12 @@ public final class JpegExtractor implements Extractor {
   private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker
   private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/";
 
+  /**
+   * The identifier to use for the image track. Chosen to avoid colliding with track IDs used by
+   * {@link Mp4Extractor} for motion photos.
+   */
+  private static final int IMAGE_TRACK_ID = 1024;
+
   private final ParsableByteArray scratch;
 
   private @MonotonicNonNull ExtractorOutput extractorOutput;
@@ -72,11 +80,16 @@ public final class JpegExtractor implements Extractor {
   @State private int state;
   private int marker;
   private int segmentLength;
+  private long mp4StartPosition;
 
   @Nullable private MotionPhotoMetadata motionPhotoMetadata;
+  private @MonotonicNonNull ExtractorInput lastExtractorInput;
+  private @MonotonicNonNull StartOffsetExtractorInput mp4ExtractorStartOffsetExtractorInput;
+  private @MonotonicNonNull Mp4Extractor mp4Extractor;
 
   public JpegExtractor() {
     scratch = new ParsableByteArray(JPEG_EXIF_HEADER_LENGTH);
+    mp4StartPosition = C.POSITION_UNSET;
   }
 
   @Override
@@ -109,12 +122,25 @@ public final class JpegExtractor implements Extractor {
         readSegment(input);
         return RESULT_CONTINUE;
       case STATE_SNIFFING_MOTION_PHOTO_VIDEO:
-        if (input.getPosition() != checkNotNull(motionPhotoMetadata).videoStartPosition) {
-          seekPosition.position = motionPhotoMetadata.videoStartPosition;
+        if (input.getPosition() != mp4StartPosition) {
+          seekPosition.position = mp4StartPosition;
           return RESULT_SEEK;
         }
         sniffMotionPhotoVideo(input);
         return RESULT_CONTINUE;
+      case STATE_READING_MOTION_PHOTO_VIDEO:
+        if (mp4ExtractorStartOffsetExtractorInput == null || input != lastExtractorInput) {
+          lastExtractorInput = input;
+          mp4ExtractorStartOffsetExtractorInput =
+              new StartOffsetExtractorInput(input, mp4StartPosition);
+        }
+        @ReadResult
+        int readResult =
+            checkNotNull(mp4Extractor).read(mp4ExtractorStartOffsetExtractorInput, seekPosition);
+        if (readResult == RESULT_SEEK) {
+          seekPosition.position += mp4StartPosition;
+        }
+        return readResult;
       case STATE_ENDED:
         return RESULT_END_OF_INPUT;
       default:
@@ -124,24 +150,29 @@ public final class JpegExtractor implements Extractor {
 
   @Override
   public void seek(long position, long timeUs) {
-    state = STATE_READING_MARKER;
+    if (position == 0) {
+      state = STATE_READING_MARKER;
+    } else if (state == STATE_READING_MOTION_PHOTO_VIDEO) {
+      checkNotNull(mp4Extractor).seek(position, timeUs);
+    }
   }
 
   @Override
   public void release() {
-    // Do nothing.
+    if (mp4Extractor != null) {
+      mp4Extractor.release();
+    }
   }
 
   private void readMarker(ExtractorInput input) throws IOException {
-    scratch.reset(2);
+    scratch.reset(/* limit= */ 2);
     input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
     marker = scratch.readUnsignedShort();
     if (marker == MARKER_SOS) { // Start of scan.
-      if (motionPhotoMetadata != null) {
+      if (mp4StartPosition != C.POSITION_UNSET) {
         state = STATE_SNIFFING_MOTION_PHOTO_VIDEO;
       } else {
-        outputTracks();
-        state = STATE_ENDED;
+        endReadingWithImageTrack();
       }
     } else if ((marker < 0xFFD0 || marker > 0xFFD9) && marker != 0xFF01) {
       state = STATE_READING_SEGMENT_LENGTH;
@@ -164,6 +195,9 @@ public final class JpegExtractor implements Extractor {
         @Nullable String xmpString = payload.readNullTerminatedString();
         if (xmpString != null) {
           motionPhotoMetadata = getMotionPhotoMetadata(xmpString, input.getLength());
+          if (motionPhotoMetadata != null) {
+            mp4StartPosition = motionPhotoMetadata.videoStartPosition;
+          }
         }
       }
     } else {
@@ -178,29 +212,41 @@ public final class JpegExtractor implements Extractor {
         input.peekFully(
             scratch.getData(), /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true);
     if (!peekedData) {
-      outputTracks();
+      endReadingWithImageTrack();
     } else {
       input.resetPeekPosition();
-      long mp4StartPosition = input.getPosition();
-      StartOffsetExtractorInput mp4ExtractorInput =
+      if (mp4Extractor == null) {
+        mp4Extractor = new Mp4Extractor();
+      }
+      mp4ExtractorStartOffsetExtractorInput =
           new StartOffsetExtractorInput(input, mp4StartPosition);
-      Mp4Extractor mp4Extractor = new Mp4Extractor();
-      if (mp4Extractor.sniff(mp4ExtractorInput)) {
-        outputTracks(checkNotNull(motionPhotoMetadata));
+      if (mp4Extractor.sniff(mp4ExtractorStartOffsetExtractorInput)) {
+        mp4Extractor.init(
+            new StartOffsetExtractorOutput(mp4StartPosition, checkNotNull(extractorOutput)));
+        startReadingMotionPhoto();
       } else {
-        outputTracks();
+        endReadingWithImageTrack();
       }
     }
+  }
+
+  private void startReadingMotionPhoto() {
+    outputImageTrack(checkNotNull(motionPhotoMetadata));
+    state = STATE_READING_MOTION_PHOTO_VIDEO;
+  }
+
+  private void endReadingWithImageTrack() {
+    outputImageTrack();
+    checkNotNull(extractorOutput).endTracks();
+    extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
     state = STATE_ENDED;
   }
 
-  private void outputTracks(Metadata.Entry... metadataEntries) {
+  private void outputImageTrack(Metadata.Entry... metadataEntries) {
     TrackOutput imageTrackOutput =
-        checkNotNull(extractorOutput).track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
+        checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE);
     imageTrackOutput.format(
         new Format.Builder().setMetadata(new Metadata(metadataEntries)).build());
-    extractorOutput.endTracks();
-    extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
   }
 
   /**

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/jpeg/StartOffsetExtractorInput.java

@@ -15,7 +15,7 @@
  */
 package com.google.android.exoplayer2.extractor.jpeg;
 
-import static com.google.android.exoplayer2.util.Assertions.checkState;
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 
 import com.google.android.exoplayer2.extractor.ExtractorInput;
 import com.google.android.exoplayer2.extractor.ForwardingExtractorInput;
@@ -38,10 +38,12 @@ import com.google.android.exoplayer2.extractor.ForwardingExtractorInput;
    * @param input The extractor input to wrap. The reading position must be at or after the start
    *     offset, otherwise data could be read from before the start offset.
    * @param startOffset The offset from which this extractor input provides data, in bytes.
+   * @throws IllegalArgumentException Thrown if the start offset is before the current reading
+   *     position.
    */
   public StartOffsetExtractorInput(ExtractorInput input, long startOffset) {
     super(input);
-    checkState(input.getPosition() >= startOffset);
+    checkArgument(input.getPosition() >= startOffset);
     this.startOffset = startOffset;
   }
 

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/jpeg/StartOffsetExtractorOutput.java

+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.extractor.jpeg;
+
+import com.google.android.exoplayer2.extractor.ExtractorOutput;
+import com.google.android.exoplayer2.extractor.SeekMap;
+import com.google.android.exoplayer2.extractor.SeekPoint;
+import com.google.android.exoplayer2.extractor.TrackOutput;
+
+/**
+ * An extractor output that wraps another extractor output and applies a give start byte offset to
+ * seek positions.
+ *
+ * <p>This is useful for extracting from a container that's concatenated after some prefix data but
+ * where the container's extractor doesn't handle a non-zero start offset (for example, because it
+ * seeks to absolute positions read from the container data).
+ */
+public final class StartOffsetExtractorOutput implements ExtractorOutput {
+
+  private final long startOffset;
+  private final ExtractorOutput extractorOutput;
+
+  /** Creates a new wrapper reading from the given start byte offset. */
+  public StartOffsetExtractorOutput(long startOffset, ExtractorOutput extractorOutput) {
+    this.startOffset = startOffset;
+    this.extractorOutput = extractorOutput;
+  }
+
+  @Override
+  public TrackOutput track(int id, int type) {
+    return extractorOutput.track(id, type);
+  }
+
+  @Override
+  public void endTracks() {
+    extractorOutput.endTracks();
+  }
+
+  @Override
+  public void seekMap(SeekMap seekMap) {
+    extractorOutput.seekMap(
+        new SeekMap() {
+          @Override
+          public boolean isSeekable() {
+            return seekMap.isSeekable();
+          }
+
+          @Override
+          public long getDurationUs() {
+            return seekMap.getDurationUs();
+          }
+
+          @Override
+          public SeekPoints getSeekPoints(long timeUs) {
+            SeekPoints seekPoints = seekMap.getSeekPoints(timeUs);
+            return new SeekPoints(
+                new SeekPoint(seekPoints.first.timeUs, seekPoints.first.position + startOffset),
+                new SeekPoint(seekPoints.second.timeUs, seekPoints.second.position + startOffset));
+          }
+        });
+  }
+}

testdata/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg.0.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg.unknown_length.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg.0.dump

 seekMap:
-  isSeekable = false
+  isSeekable = true
   duration = UNSET TIME
-  getPosition(0) = [[timeUs=0, position=0]]
+  getPosition(0) = [[timeUs=0, position=131582]]
+  getPosition(1) = [[timeUs=0, position=131582]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg.unknown_length.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/pixel-motion-photo-video-removed-shortened.jpg.0.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/pixel-motion-photo-video-removed-shortened.jpg.unknown_length.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/ss-motion-photo-shortened.jpg.0.dump

 seekMap:
-  isSeekable = false
+  isSeekable = true
   duration = UNSET TIME
-  getPosition(0) = [[timeUs=0, position=0]]
+  getPosition(0) = [[timeUs=0, position=20345]]
+  getPosition(1) = [[timeUs=0, position=20345]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0:

testdata/src/test/assets/extractordumps/jpeg/ss-motion-photo-shortened.jpg.unknown_length.dump

@@ -3,7 +3,7 @@ seekMap:
   duration = UNSET TIME
   getPosition(0) = [[timeUs=0, position=0]]
 numberOfTracks = 1
-track 0:
+track 1024:
   total output bytes = 0
   sample count = 0
   format 0: