Extract SEF slow motion cues as Metadata

PiperOrigin-RevId: 339307746

Extract SEF slow motion cues as Metadata
PiperOrigin-RevId: 339307746
c0a0708f · samrobinson · Oliver Woodman · 1c4653f7 · c0a0708f · c0a0708f
Commit c0a0708f authored Oct 27, 2020 by samrobinson Committed by Oliver Woodman Nov 02, 2020
Showing with 279 additions and 11 deletions
library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/SefReader.java
--- a/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
@@ -48,7 +48,7 @@ public final class MetadataRetriever {
   *
   * <p>This is equivalent to using {@link #retrieveMetadata(MediaSourceFactory, MediaItem)} with a
   * {@link DefaultMediaSourceFactory} and a {@link DefaultExtractorsFactory} with {@link
-   * Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} set.
+   * Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} and {@link Mp4Extractor#FLAG_READ_SEF_DATA} set.
   *
   * @param context The {@link Context}.
   * @param mediaItem The {@link MediaItem} whose metadata should be retrieved.
@@ -58,7 +58,8 @@ public final class MetadataRetriever {
      Context context, MediaItem mediaItem) {
    ExtractorsFactory extractorsFactory =
        new DefaultExtractorsFactory()
-            .setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA);
+            .setMp4ExtractorFlags(
+                Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA | Mp4Extractor.FLAG_READ_SEF_DATA);
    MediaSourceFactory mediaSourceFactory =
        new DefaultMediaSourceFactory(context, extractorsFactory);
    return retrieveMetadata(mediaSourceFactory, mediaItem);

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
@@ -41,6 +41,7 @@ import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
 import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.metadata.mp4.MotionPhotoMetadata;
+import com.google.android.exoplayer2.metadata.mp4.SefSlowMotion;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
@@ -65,17 +66,20 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  /**
   * Flags controlling the behavior of the extractor. Possible flag values are {@link
-   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS} and {@link #FLAG_READ_MOTION_PHOTO_METADATA}.
+   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}, {@link #FLAG_READ_MOTION_PHOTO_METADATA} and {@link
+   * #FLAG_READ_SEF_DATA}.
   */
  @Documented
  @Retention(RetentionPolicy.SOURCE)
  @IntDef(
      flag = true,
-      value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS, FLAG_READ_MOTION_PHOTO_METADATA})
+      value = {
+        FLAG_WORKAROUND_IGNORE_EDIT_LISTS,
+        FLAG_READ_MOTION_PHOTO_METADATA,
+        FLAG_READ_SEF_DATA
+      })
  public @interface Flags {}
-  /**
+  /** Flag to ignore any edit lists in the stream. */
-   * Flag to ignore any edit lists in the stream.
-   */
  public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1;
  /**
   * Flag to extract {@link MotionPhotoMetadata} from HEIC motion photos following the Google Photos
@@ -85,16 +89,27 @@ public final class Mp4Extractor implements Extractor, SeekMap {
   * retrieval use cases.
   */
  public static final int FLAG_READ_MOTION_PHOTO_METADATA = 1 << 1;
+  /**
+   * Flag to extract {@link SefSlowMotion} metadata from Samsung Extension Format (SEF) slow motion
+   * videos.
+   */
+  public static final int FLAG_READ_SEF_DATA = 1 << 2;
  /** Parser states. */
  @Documented
  @Retention(RetentionPolicy.SOURCE)
-  @IntDef({STATE_READING_ATOM_HEADER, STATE_READING_ATOM_PAYLOAD, STATE_READING_SAMPLE})
+  @IntDef({
+    STATE_READING_ATOM_HEADER,
+    STATE_READING_ATOM_PAYLOAD,
+    STATE_READING_SAMPLE,
+    STATE_READING_SEF,
+  })
  private @interface State {}
  private static final int STATE_READING_ATOM_HEADER = 0;
  private static final int STATE_READING_ATOM_PAYLOAD = 1;
  private static final int STATE_READING_SAMPLE = 2;
+  private static final int STATE_READING_SEF = 3;
  /** Supported file types. */
  @Documented
@@ -127,6 +142,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
  private final ParsableByteArray atomHeader;
  private final ArrayDeque<ContainerAtom> containerAtoms;
+  private final SefReader sefReader;
+  private final List<Metadata.Entry> slowMotionMetadataEntries;
  @State private int parserState;
  private int atomType;
@@ -153,7 +170,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
   * Creates a new extractor for unfragmented MP4 streams.
   */
  public Mp4Extractor() {
-    this(0);
+    this(/* flags= */ 0);
  }
  /**
@@ -164,6 +181,10 @@ public final class Mp4Extractor implements Extractor, SeekMap {
   */
  public Mp4Extractor(@Flags int flags) {
    this.flags = flags;
+    parserState =
+        ((flags & FLAG_READ_SEF_DATA) != 0) ? STATE_READING_SEF : STATE_READING_ATOM_HEADER;
+    sefReader = new SefReader();
+    slowMotionMetadataEntries = new ArrayList<>();
    atomHeader = new ParsableByteArray(Atom.LONG_HEADER_SIZE);
    containerAtoms = new ArrayDeque<>();
    nalStartCode = new ParsableByteArray(NalUnitUtil.NAL_START_CODE);
@@ -192,7 +213,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    sampleBytesWritten = 0;
    sampleCurrentNalBytesRemaining = 0;
    if (position == 0) {
-      enterReadingAtomHeaderState();
+      // Reading the SEF data occurs before normal MP4 parsing. Therefore we can not transition to
+      // reading the atom header until that has completed.
+      if (parserState != STATE_READING_SEF) {
+        enterReadingAtomHeaderState();
+      } else {
+        sefReader.reset();
+        slowMotionMetadataEntries.clear();
+      }
    } else if (tracks != null) {
      updateSampleIndices(timeUs);
    }
@@ -219,6 +247,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
          break;
        case STATE_READING_SAMPLE:
          return readSample(input, seekPosition);
+        case STATE_READING_SEF:
+          return readSefData(input, seekPosition);
        default:
          throw new IllegalStateException();
      }
@@ -396,6 +426,15 @@ public final class Mp4Extractor implements Extractor, SeekMap {
    return seekRequired && parserState != STATE_READING_SAMPLE;
  }
+  @ReadResult
+  private int readSefData(ExtractorInput input, PositionHolder seekPosition) throws IOException {
+    @ReadResult int result = sefReader.read(input, seekPosition, slowMotionMetadataEntries);
+    if (result == RESULT_SEEK && seekPosition.position == 0) {
+      enterReadingAtomHeaderState();
+    }
+    return result;
+  }
  private void processAtomEnded(long atomEndPosition) throws ParserException {
    while (!containerAtoms.isEmpty() && containerAtoms.peek().endPosition == atomEndPosition) {
      Atom.ContainerAtom containerAtom = containerAtoms.pop();
@@ -474,8 +513,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
        float frameRate = trackSampleTable.sampleCount / (trackDurationUs / 1000000f);
        formatBuilder.setFrameRate(frameRate);
      }
      MetadataUtil.setFormatMetadata(
-          track.type, udtaMetadata, mdtaMetadata, gaplessInfoHolder, formatBuilder);
+          track.type,
+          udtaMetadata,
+          mdtaMetadata,
+          gaplessInfoHolder,
+          formatBuilder,
+          /* additionalEntries...= */ slowMotionMetadataEntries.toArray(new Metadata.Entry[0]));
      mp4Track.trackOutput.format(formatBuilder.build());
      if (track.type == C.TRACK_TYPE_VIDEO && firstVideoTrackIndex == C.INDEX_UNSET) {

--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/SefReader.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/SefReader.java
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.extractor.mp4;
+import static com.google.android.exoplayer2.extractor.Extractor.RESULT_SEEK;
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import androidx.annotation.IntDef;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.ParserException;
+import com.google.android.exoplayer2.extractor.Extractor;
+import com.google.android.exoplayer2.extractor.ExtractorInput;
+import com.google.android.exoplayer2.extractor.PositionHolder;
+import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.SefSlowMotion;
+import com.google.android.exoplayer2.util.ParsableByteArray;
+import com.google.common.base.Splitter;
+import java.io.IOException;
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+/**
+ * Reads Samsung Extension Format (SEF) metadata.
+ *
+ * <p>To be used in conjunction with {@link Mp4Extractor}.
+ */
+/* package */ final class SefReader {
+  /** Reader states. */
+  @Documented
+  @Retention(RetentionPolicy.SOURCE)
+  @IntDef({
+    STATE_SHOULD_CHECK_FOR_SEF,
+    STATE_CHECKING_FOR_SEF,
+    STATE_READING_SDRS,
+    STATE_READING_SEF_DATA
+  })
+  private @interface State {}
+  private static final int STATE_SHOULD_CHECK_FOR_SEF = 0;
+  private static final int STATE_CHECKING_FOR_SEF = 1;
+  private static final int STATE_READING_SDRS = 2;
+  private static final int STATE_READING_SEF_DATA = 3;
+  /** Supported data types. */
+  @Documented
+  @Retention(RetentionPolicy.SOURCE)
+  @IntDef({TYPE_SLOW_MOTION_DATA})
+  private @interface DataType {}
+  private static final int TYPE_SLOW_MOTION_DATA = 0x0890;
+  private static final String TAG = "SefReader";
+  // Hex representation of `SEFT` (in ASCII). This is the last byte of a file that has Samsung
+  // Extension Format (SEF) data.
+  private static final int SAMSUNG_TAIL_SIGNATURE = 0x53454654;
+  // Start signature (4 bytes), SEF version (4 bytes), SDR count (4 bytes).
+  private static final int TAIL_HEADER_LENGTH = 12;
+  // Tail offset (4 bytes), tail signature (4 bytes).
+  private static final int TAIL_FOOTER_LENGTH = 8;
+  private static final int LENGTH_OF_ONE_SDR = 12;
+  private final List<DataReference> dataReferences;
+  @State private int readerState;
+  private int tailLength;
+  public SefReader() {
+    dataReferences = new ArrayList<>();
+    readerState = STATE_SHOULD_CHECK_FOR_SEF;
+  }
+  public void reset() {
+    dataReferences.clear();
+    readerState = STATE_SHOULD_CHECK_FOR_SEF;
+  }
+  @Extractor.ReadResult
+  public int read(
+      ExtractorInput input,
+      PositionHolder seekPosition,
+      List<Metadata.Entry> slowMotionMetadataEntries)
+      throws IOException {
+    switch (readerState) {
+      case STATE_SHOULD_CHECK_FOR_SEF:
+        long inputLength = input.getLength();
+        seekPosition.position =
+            inputLength == C.LENGTH_UNSET || inputLength < TAIL_FOOTER_LENGTH
+                ? 0
+                : inputLength - TAIL_FOOTER_LENGTH;
+        readerState = STATE_CHECKING_FOR_SEF;
+        break;
+      case STATE_CHECKING_FOR_SEF:
+        checkForSefData(input, seekPosition);
+        break;
+      case STATE_READING_SDRS:
+        readSdrs(input, seekPosition);
+        break;
+      case STATE_READING_SEF_DATA:
+        readSefData(input, slowMotionMetadataEntries);
+        seekPosition.position = 0;
+        break;
+      default:
+        throw new IllegalStateException();
+    }
+    return RESULT_SEEK;
+  }
+  private void checkForSefData(ExtractorInput input, PositionHolder seekPosition)
+      throws IOException {
+    ParsableByteArray scratch = new ParsableByteArray(/* limit= */ TAIL_FOOTER_LENGTH);
+    input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ TAIL_FOOTER_LENGTH);
+    tailLength = scratch.readLittleEndianInt() + TAIL_FOOTER_LENGTH;
+    if (scratch.readInt() != SAMSUNG_TAIL_SIGNATURE) {
+      seekPosition.position = 0;
+      return;
+    }
+    // input.getPosition is at the very end of the tail, so jump forward by sefTailLength, but
+    // account for the tail header, which needs to be ignored.
+    seekPosition.position = input.getPosition() - (tailLength - TAIL_HEADER_LENGTH);
+    readerState = STATE_READING_SDRS;
+  }
+  private void readSdrs(ExtractorInput input, PositionHolder seekPosition) throws IOException {
+    long streamLength = input.getLength();
+    int sdrsLength = tailLength - TAIL_HEADER_LENGTH - TAIL_FOOTER_LENGTH;
+    ParsableByteArray scratch = new ParsableByteArray(/* limit= */ sdrsLength);
+    input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ sdrsLength);
+    for (int i = 0; i < sdrsLength / LENGTH_OF_ONE_SDR; i++) {
+      scratch.skipBytes(2); // SDR data sub info flag and reserved bits (2).
+      @DataType int dataType = scratch.readLittleEndianShort();
+      if (dataType == TYPE_SLOW_MOTION_DATA) {
+        // The read int is the distance from the tail info to the start of the metadata.
+        // Calculated as an offset from the start by working backwards.
+        long startOffset = streamLength - tailLength - scratch.readLittleEndianInt();
+        int size = scratch.readLittleEndianInt();
+        dataReferences.add(new DataReference(dataType, startOffset, size));
+      } else {
+        scratch.skipBytes(8); // startPosition (4), size (4).
+      }
+    }
+    if (dataReferences.isEmpty()) {
+      seekPosition.position = 0;
+      return;
+    }
+    Collections.sort(dataReferences, (o1, o2) -> Long.compare(o1.startOffset, o2.startOffset));
+    readerState = STATE_READING_SEF_DATA;
+    seekPosition.position = dataReferences.get(0).startOffset;
+  }
+  private void readSefData(ExtractorInput input, List<Metadata.Entry> slowMotionMetadataEntries)
+      throws IOException {
+    checkNotNull(dataReferences);
+    Splitter splitter = Splitter.on(':');
+    int totalDataLength = (int) (input.getLength() - input.getPosition() - tailLength);
+    ParsableByteArray scratch = new ParsableByteArray(/* limit= */ totalDataLength);
+    input.readFully(scratch.getData(), 0, totalDataLength);
+    int totalDataReferenceBytesConsumed = 0;
+    for (int i = 0; i < dataReferences.size(); i++) {
+      DataReference dataReference = dataReferences.get(i);
+      if (dataReference.dataType == TYPE_SLOW_MOTION_DATA) {
+        scratch.skipBytes(23); // data type (2), data sub info (2), name len (4), name (15).
+        List<SefSlowMotion.Segment> segments = new ArrayList<>();
+        int dataReferenceEndPosition = totalDataReferenceBytesConsumed + dataReference.size;
+        while (scratch.getPosition() < dataReferenceEndPosition) {
+          @Nullable String data = scratch.readDelimiterTerminatedString('*');
+          List<String> values = splitter.splitToList(checkNotNull(data));
+          if (values.size() != 3) {
+            throw new ParserException();
+          }
+          try {
+            int startTimeMs = Integer.parseInt(values.get(0));
+            int endTimeMs = Integer.parseInt(values.get(1));
+            int speedMode = Integer.parseInt(values.get(2));
+            int speedDivisor = 1 << (speedMode - 1);
+            segments.add(new SefSlowMotion.Segment(startTimeMs, endTimeMs, speedDivisor));
+          } catch (NumberFormatException e) {
+            throw new ParserException(e);
+          }
+        }
+        totalDataReferenceBytesConsumed += dataReference.size;
+        slowMotionMetadataEntries.add(new SefSlowMotion(segments));
+      }
+    }
+  }
+  private static final class DataReference {
+    @DataType public final int dataType;
+    public final long startOffset;
+    public final int size;
+    public DataReference(@DataType int dataType, long startOffset, int size) {
+      this.dataType = dataType;
+      this.startOffset = startOffset;
+      this.size = size;
+    }
+  }
+}