FMP4: Fix output of mixed v0 and v1 emsg samples

Issue: google/ExoPlayer#9996
#minor-release
PiperOrigin-RevId: 430773329

RELEASENOTES.md

@@ -6,7 +6,10 @@
     *   Flatten `TrackSelectionOverrides` class into `TrackSelectionParameters`,
     *   Flatten `TrackSelectionOverrides` class into `TrackSelectionParameters`,
         and promote `TrackSelectionOverride` to a top level class.
         and promote `TrackSelectionOverride` to a top level class.
 *   Extractors:
 *   Extractors:
-    *   Opus DiscardPadding in Matroska files.
+    *   Matroska: Parse `DiscardPadding` for Opus tracks.
+    *   FMP4: Fix issue where emsg sample metadata could be output in the wrong
+        order for streams containing both v0 and v1 emsg atoms
+        ([#9996](https://github.com/google/ExoPlayer/issues/9996)).
 *   RTSP:
 *   RTSP:
     *   Add RTP reader for HEVC
     *   Add RTP reader for HEVC
         ([#36](https://github.com/androidx/media/pull/36))
         ([#36](https://github.com/androidx/media/pull/36))

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/FragmentedMp4Extractor.java

@@ -666,14 +666,23 @@ public class FragmentedMp4Extractor implements Extractor {
       emsgTrackOutput.sampleData(encodedEventMessage, sampleSize);
       emsgTrackOutput.sampleData(encodedEventMessage, sampleSize);
     }
     }
 
 
-    // Output the sample metadata. This is made a little complicated because emsg-v0 atoms
-    // have presentation time *delta* while v1 atoms have absolute presentation time.
+    // Output the sample metadata.
     if (sampleTimeUs == C.TIME_UNSET) {
     if (sampleTimeUs == C.TIME_UNSET) {
-      // We need the first sample timestamp in the segment before we can output the metadata.
+      // We're processing a v0 emsg atom, which contains a presentation time delta, and cannot yet
+      // calculate its absolute sample timestamp. Defer outputting the metadata until we can.
       pendingMetadataSampleInfos.addLast(
       pendingMetadataSampleInfos.addLast(
-          new MetadataSampleInfo(presentationTimeDeltaUs, sampleSize));
+          new MetadataSampleInfo(
+              presentationTimeDeltaUs, /* sampleTimeIsRelative= */ true, sampleSize));
+      pendingMetadataSampleBytes += sampleSize;
+    } else if (!pendingMetadataSampleInfos.isEmpty()) {
+      // We also need to defer outputting metadata if pendingMetadataSampleInfos is non-empty, else
+      // we will output metadata for samples in the wrong order. See:
+      // https://github.com/google/ExoPlayer/issues/9996.
+      pendingMetadataSampleInfos.addLast(
+          new MetadataSampleInfo(sampleTimeUs, /* sampleTimeIsRelative= */ false, sampleSize));
       pendingMetadataSampleBytes += sampleSize;
       pendingMetadataSampleBytes += sampleSize;
     } else {
     } else {
+      // We can output the sample metadata immediately.
       if (timestampAdjuster != null) {
       if (timestampAdjuster != null) {
         sampleTimeUs = timestampAdjuster.adjustSampleTimestamp(sampleTimeUs);
         sampleTimeUs = timestampAdjuster.adjustSampleTimestamp(sampleTimeUs);
       }
       }
@@ -1459,19 +1468,30 @@ public class FragmentedMp4Extractor implements Extractor {
     return true;
     return true;
   }
   }
 
 
+  /**
+   * Called immediately after outputting a non-metadata sample, to output any pending metadata
+   * samples.
+   *
+   * @param sampleTimeUs The timestamp of the non-metadata sample that was just output.
+   */
   private void outputPendingMetadataSamples(long sampleTimeUs) {
   private void outputPendingMetadataSamples(long sampleTimeUs) {
     while (!pendingMetadataSampleInfos.isEmpty()) {
     while (!pendingMetadataSampleInfos.isEmpty()) {
-      MetadataSampleInfo sampleInfo = pendingMetadataSampleInfos.removeFirst();
-      pendingMetadataSampleBytes -= sampleInfo.size;
-      long metadataTimeUs = sampleTimeUs + sampleInfo.presentationTimeDeltaUs;
+      MetadataSampleInfo metadataSampleInfo = pendingMetadataSampleInfos.removeFirst();
+      pendingMetadataSampleBytes -= metadataSampleInfo.size;
+      long metadataSampleTimeUs = metadataSampleInfo.sampleTimeUs;
+      if (metadataSampleInfo.sampleTimeIsRelative) {
+        // The metadata sample timestamp is relative to the timestamp of the non-metadata sample
+        // that was just output. Make it absolute.
+        metadataSampleTimeUs += sampleTimeUs;
+      }
       if (timestampAdjuster != null) {
       if (timestampAdjuster != null) {
-        metadataTimeUs = timestampAdjuster.adjustSampleTimestamp(metadataTimeUs);
+        metadataSampleTimeUs = timestampAdjuster.adjustSampleTimestamp(metadataSampleTimeUs);
       }
       }
       for (TrackOutput emsgTrackOutput : emsgTrackOutputs) {
       for (TrackOutput emsgTrackOutput : emsgTrackOutputs) {
         emsgTrackOutput.sampleMetadata(
         emsgTrackOutput.sampleMetadata(
-            metadataTimeUs,
+            metadataSampleTimeUs,
             C.BUFFER_FLAG_KEY_FRAME,
             C.BUFFER_FLAG_KEY_FRAME,
-            sampleInfo.size,
+            metadataSampleInfo.size,
             pendingMetadataSampleBytes,
             pendingMetadataSampleBytes,
             null);
             null);
       }
       }
@@ -1577,11 +1597,13 @@ public class FragmentedMp4Extractor implements Extractor {
   /** Holds data corresponding to a metadata sample. */
   /** Holds data corresponding to a metadata sample. */
   private static final class MetadataSampleInfo {
   private static final class MetadataSampleInfo {
 
 
-    public final long presentationTimeDeltaUs;
+    public final long sampleTimeUs;
+    public final boolean sampleTimeIsRelative;
     public final int size;
     public final int size;
 
 
-    public MetadataSampleInfo(long presentationTimeDeltaUs, int size) {
-      this.presentationTimeDeltaUs = presentationTimeDeltaUs;
+    public MetadataSampleInfo(long sampleTimeUs, boolean sampleTimeIsRelative, int size) {
+      this.sampleTimeUs = sampleTimeUs;
+      this.sampleTimeIsRelative = sampleTimeIsRelative;
       this.size = size;
       this.size = size;
     }
     }
   }
   }