Skip to content

SDK / exoplayer

Go to a project
Commit 21c1b8ca by Oliver Woodman
Options

Add basic handling for edit lists in MP4 streams. 

Issue: #874
parent b03278f2
Inline Side-by-side
Showing with 142 additions and 17 deletions
library/src/main/java/com/google/android/exoplayer/extractor/mp4/Atom.java
...@@ -80,6 +80,8 @@ import java.util.List; ...@@ -80,6 +80,8 @@ import java.util.List;
public static final int TYPE_traf = Util.getIntegerCodeForString("traf"); public static final int TYPE_traf = Util.getIntegerCodeForString("traf");
public static final int TYPE_mvex = Util.getIntegerCodeForString("mvex"); public static final int TYPE_mvex = Util.getIntegerCodeForString("mvex");
public static final int TYPE_tkhd = Util.getIntegerCodeForString("tkhd"); public static final int TYPE_tkhd = Util.getIntegerCodeForString("tkhd");
public static final int TYPE_edts = Util.getIntegerCodeForString("edts");
public static final int TYPE_elst = Util.getIntegerCodeForString("elst");
public static final int TYPE_mdhd = Util.getIntegerCodeForString("mdhd"); public static final int TYPE_mdhd = Util.getIntegerCodeForString("mdhd");
public static final int TYPE_hdlr = Util.getIntegerCodeForString("hdlr"); public static final int TYPE_hdlr = Util.getIntegerCodeForString("hdlr");
public static final int TYPE_stsd = Util.getIntegerCodeForString("stsd"); public static final int TYPE_stsd = Util.getIntegerCodeForString("stsd");
......
library/src/main/java/com/google/android/exoplayer/extractor/mp4/AtomParsers.java
...@@ -32,7 +32,9 @@ import java.util.ArrayList; ...@@ -32,7 +32,9 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
/** Utility methods for parsing MP4 format atom payloads according to ISO 14496-12. */ /**
* Utility methods for parsing MP4 format atom payloads according to ISO 14496-12.
*/
/* package */ final class AtomParsers { /* package */ final class AtomParsers {
/** /**
...@@ -65,9 +67,11 @@ import java.util.List; ...@@ -65,9 +67,11 @@ import java.util.List;
Pair<Long, String> mdhdData = parseMdhd(mdia.getLeafAtomOfType(Atom.TYPE_mdhd).data); Pair<Long, String> mdhdData = parseMdhd(mdia.getLeafAtomOfType(Atom.TYPE_mdhd).data);
StsdData stsdData = parseStsd(stbl.getLeafAtomOfType(Atom.TYPE_stsd).data, tkhdData.id, StsdData stsdData = parseStsd(stbl.getLeafAtomOfType(Atom.TYPE_stsd).data, tkhdData.id,
durationUs, tkhdData.rotationDegrees, mdhdData.second); durationUs, tkhdData.rotationDegrees, mdhdData.second);
Pair<long[], long[]> edtsData = parseEdts(trak.getContainerAtomOfType(Atom.TYPE_edts));
return stsdData.mediaFormat == null ? null return stsdData.mediaFormat == null ? null
: new Track(tkhdData.id, trackType, mdhdData.first, durationUs, stsdData.mediaFormat, : new Track(tkhdData.id, trackType, mdhdData.first, movieTimescale, durationUs,
stsdData.trackEncryptionBoxes, stsdData.nalUnitLengthFieldLength); stsdData.mediaFormat, stsdData.trackEncryptionBoxes, stsdData.nalUnitLengthFieldLength,
edtsData.first, edtsData.second);
} }
/** /**
...@@ -240,15 +244,78 @@ import java.util.List; ...@@ -240,15 +244,78 @@ import java.util.List;
} }
} }
Util.scaleLargeTimestampsInPlace(timestamps, 1000000, track.timescale);
// Check all the expected samples have been seen. // Check all the expected samples have been seen.
Assertions.checkArgument(remainingSynchronizationSamples == 0); Assertions.checkArgument(remainingSynchronizationSamples == 0);
Assertions.checkArgument(remainingSamplesAtTimestampDelta == 0); Assertions.checkArgument(remainingSamplesAtTimestampDelta == 0);
Assertions.checkArgument(remainingSamplesInChunk == 0); Assertions.checkArgument(remainingSamplesInChunk == 0);
Assertions.checkArgument(remainingTimestampDeltaChanges == 0); Assertions.checkArgument(remainingTimestampDeltaChanges == 0);
Assertions.checkArgument(remainingTimestampOffsetChanges == 0); Assertions.checkArgument(remainingTimestampOffsetChanges == 0);
return new TrackSampleTable(offsets, sizes, maximumSize, timestamps, flags);
if (track.editListDurations == null) {
Util.scaleLargeTimestampsInPlace(timestamps, C.MICROS_PER_SECOND, track.timescale);
return new TrackSampleTable(offsets, sizes, maximumSize, timestamps, flags);
}
// See the BMFF spec (ISO 14496-12) subsection 8.6.6. Edit lists that truncate audio and
// require prerolling from a sync sample after reordering are not supported. This
// implementation handles simple discarding/delaying of samples. The extractor may place
// further restrictions on what edited streams are playable.
// Count the number of samples after applying edits.
int editedSampleCount = 0;
int nextSampleIndex = 0;
boolean copyMetadata = false;
for (int i = 0; i < track.editListDurations.length; i++) {
long mediaTime = track.editListMediaTimes[i];
if (mediaTime != -1) {
long duration = Util.scaleLargeTimestamp(track.editListDurations[i], track.timescale,
track.movieTimescale);
int startIndex = Util.binarySearchCeil(timestamps, mediaTime, true, true);
int endIndex = Util.binarySearchCeil(timestamps, mediaTime + duration, true, false);
editedSampleCount += endIndex - startIndex;
copyMetadata |= nextSampleIndex != startIndex;
nextSampleIndex = endIndex;
}
}
copyMetadata |= editedSampleCount != sampleCount;
// Calculate edited sample timestamps and update the corresponding metadata arrays.
long[] editedOffsets = copyMetadata ? new long[editedSampleCount] : offsets;
int[] editedSizes = copyMetadata ? new int[editedSampleCount] : sizes;
int editedMaximumSize = copyMetadata ? 0 : maximumSize;
int[] editedFlags = copyMetadata ? new int[editedSampleCount] : flags;
long[] editedTimestamps = new long[editedSampleCount];
long pts = 0;
int sampleIndex = 0;
for (int i = 0; i < track.editListDurations.length; i++) {
long mediaTime = track.editListMediaTimes[i];
long duration = track.editListDurations[i];
if (mediaTime != -1) {
long endMediaTime = mediaTime + Util.scaleLargeTimestamp(duration, track.timescale,
track.movieTimescale);
int startIndex = Util.binarySearchCeil(timestamps, mediaTime, true, true);
int endIndex = Util.binarySearchCeil(timestamps, endMediaTime, true, false);
if (copyMetadata) {
int count = endIndex - startIndex;
System.arraycopy(offsets, startIndex, editedOffsets, sampleIndex, count);
System.arraycopy(sizes, startIndex, editedSizes, sampleIndex, count);
System.arraycopy(flags, startIndex, editedFlags, sampleIndex, count);
}
for (int j = startIndex; j < endIndex; j++) {
long ptsUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale);
long timeInSegmentUs = Util.scaleLargeTimestamp(timestamps[j] - mediaTime,
C.MICROS_PER_SECOND, track.timescale);
editedTimestamps[sampleIndex] = ptsUs + timeInSegmentUs;
if (copyMetadata && editedSizes[sampleIndex] > editedMaximumSize) {
editedMaximumSize = sizes[j];
}
sampleIndex++;
}
}
pts += duration;
}
return new TrackSampleTable(editedOffsets, editedSizes, editedMaximumSize, editedTimestamps,
editedFlags);
} }
/** /**
...@@ -541,6 +608,39 @@ import java.util.List; ...@@ -541,6 +608,39 @@ import java.util.List;
return Pair.create(initializationData, lengthSizeMinusOne + 1); return Pair.create(initializationData, lengthSizeMinusOne + 1);
} }
/**
* Parses the edts atom (defined in 14496-12 subsection 8.6.5).
*
* @param edtsAtom edts (edit box) atom to parse.
* @return Pair of edit list durations and edit list media times, or a pair of nulls if they are
* not present.
*/
private static Pair<long[], long[]> parseEdts(Atom.ContainerAtom edtsAtom) {
Atom.LeafAtom elst;
if (edtsAtom == null || (elst = edtsAtom.getLeafAtomOfType(Atom.TYPE_elst)) == null) {
return Pair.create(null, null);
}
ParsableByteArray elstData = elst.data;
elstData.setPosition(Atom.HEADER_SIZE);
int fullAtom = elstData.readInt();
int version = Atom.parseFullAtomVersion(fullAtom);
int entryCount = elstData.readUnsignedIntToInt();
long[] editListDurations = new long[entryCount];
long[] editListMediaTimes = new long[entryCount];
for (int i = 0; i < entryCount; i++) {
editListDurations[i] =
version == 1 ? elstData.readUnsignedLongToLong() : elstData.readUnsignedInt();
editListMediaTimes[i] = version == 1 ? elstData.readLong() : elstData.readInt();
int mediaRateInteger = elstData.readShort();
if (mediaRateInteger != 1) {
// The extractor does not handle dwell edits (mediaRateInteger == 0).
throw new IllegalArgumentException("Unsupported media rate.");
}
elstData.skipBytes(2);
}
return Pair.create(editListDurations, editListMediaTimes);
}
private static TrackEncryptionBox parseSinfFromParent(ParsableByteArray parent, int position, private static TrackEncryptionBox parseSinfFromParent(ParsableByteArray parent, int position,
int size) { int size) {
int childPosition = position + Atom.HEADER_SIZE; int childPosition = position + Atom.HEADER_SIZE;
......
library/src/main/java/com/google/android/exoplayer/extractor/mp4/Mp4Extractor.java
...@@ -138,11 +138,12 @@ public final class Mp4Extractor implements Extractor, SeekMap { ...@@ -138,11 +138,12 @@ public final class Mp4Extractor implements Extractor, SeekMap {
TrackSampleTable sampleTable = tracks[trackIndex].sampleTable; TrackSampleTable sampleTable = tracks[trackIndex].sampleTable;
int sampleIndex = sampleTable.getIndexOfEarlierOrEqualSynchronizationSample(timeUs); int sampleIndex = sampleTable.getIndexOfEarlierOrEqualSynchronizationSample(timeUs);
if (sampleIndex == TrackSampleTable.NO_SAMPLE) { if (sampleIndex == TrackSampleTable.NO_SAMPLE) {
// Handle the case where the requested time is before the first synchronization sample.
sampleIndex = sampleTable.getIndexOfLaterOrEqualSynchronizationSample(timeUs); sampleIndex = sampleTable.getIndexOfLaterOrEqualSynchronizationSample(timeUs);
} }
tracks[trackIndex].sampleIndex = sampleIndex; tracks[trackIndex].sampleIndex = sampleIndex;
long offset = sampleTable.offsets[tracks[trackIndex].sampleIndex]; long offset = sampleTable.offsets[sampleIndex];
if (offset < earliestSamplePosition) { if (offset < earliestSamplePosition) {
earliestSamplePosition = offset; earliestSamplePosition = offset;
} }
...@@ -386,15 +387,15 @@ public final class Mp4Extractor implements Extractor, SeekMap { ...@@ -386,15 +387,15 @@ public final class Mp4Extractor implements Extractor, SeekMap {
|| atom == Atom.TYPE_vmhd || atom == Atom.TYPE_smhd || atom == Atom.TYPE_stsd || atom == Atom.TYPE_vmhd || atom == Atom.TYPE_smhd || atom == Atom.TYPE_stsd
|| atom == Atom.TYPE_avc1 || atom == Atom.TYPE_avcC || atom == Atom.TYPE_mp4a || atom == Atom.TYPE_avc1 || atom == Atom.TYPE_avcC || atom == Atom.TYPE_mp4a
|| atom == Atom.TYPE_esds || atom == Atom.TYPE_stts || atom == Atom.TYPE_stss || atom == Atom.TYPE_esds || atom == Atom.TYPE_stts || atom == Atom.TYPE_stss
|| atom == Atom.TYPE_ctts || atom == Atom.TYPE_stsc || atom == Atom.TYPE_stsz || atom == Atom.TYPE_ctts || atom == Atom.TYPE_elst || atom == Atom.TYPE_stsc
|| atom == Atom.TYPE_stco || atom == Atom.TYPE_co64 || atom == Atom.TYPE_tkhd || atom == Atom.TYPE_stsz || atom == Atom.TYPE_stco || atom == Atom.TYPE_co64
|| atom == Atom.TYPE_s263; || atom == Atom.TYPE_tkhd || atom == Atom.TYPE_s263;
} }
/** Returns whether the extractor should parse a container atom with type {@code atom}. */ /** Returns whether the extractor should parse a container atom with type {@code atom}. */
private static boolean shouldParseContainerAtom(int atom) { private static boolean shouldParseContainerAtom(int atom) {
return atom == Atom.TYPE_moov || atom == Atom.TYPE_trak || atom == Atom.TYPE_mdia return atom == Atom.TYPE_moov || atom == Atom.TYPE_trak || atom == Atom.TYPE_mdia
|| atom == Atom.TYPE_minf || atom == Atom.TYPE_stbl; || atom == Atom.TYPE_minf || atom == Atom.TYPE_stbl || atom == Atom.TYPE_edts;
} }
private static final class Mp4Track { private static final class Mp4Track {
......
library/src/main/java/com/google/android/exoplayer/extractor/mp4/Track.java
...@@ -47,6 +47,11 @@ public final class Track { ...@@ -47,6 +47,11 @@ public final class Track {
public final long timescale; public final long timescale;
/** /**
* The movie timescale.
*/
public final long movieTimescale;
/**
* The duration of the track in microseconds, or {@link C#UNKNOWN_TIME_US} if unknown. * The duration of the track in microseconds, or {@link C#UNKNOWN_TIME_US} if unknown.
*/ */
public final long durationUs; public final long durationUs;
...@@ -62,20 +67,34 @@ public final class Track { ...@@ -62,20 +67,34 @@ public final class Track {
public final TrackEncryptionBox[] sampleDescriptionEncryptionBoxes; public final TrackEncryptionBox[] sampleDescriptionEncryptionBoxes;
/** /**
* Durations of edit list segments in the movie timescale. Null if there is no edit list.
*/
public final long[] editListDurations;
/**
* Media times for edit list segments in the track timescale. Null if there is no edit list.
*/
public final long[] editListMediaTimes;
/**
* For H264 video tracks, the length in bytes of the NALUnitLength field in each sample. -1 for * For H264 video tracks, the length in bytes of the NALUnitLength field in each sample. -1 for
* other track types. * other track types.
*/ */
public final int nalUnitLengthFieldLength; public final int nalUnitLengthFieldLength;
public Track(int id, int type, long timescale, long durationUs, MediaFormat mediaFormat, public Track(int id, int type, long timescale, long movieTimescale, long durationUs,
TrackEncryptionBox[] sampleDescriptionEncryptionBoxes, int nalUnitLengthFieldLength) { MediaFormat mediaFormat, TrackEncryptionBox[] sampleDescriptionEncryptionBoxes,
int nalUnitLengthFieldLength, long[] editListDurations, long[] editListMediaTimes) {
this.id = id; this.id = id;
this.type = type; this.type = type;
this.timescale = timescale; this.timescale = timescale;
this.movieTimescale = movieTimescale;
this.durationUs = durationUs; this.durationUs = durationUs;
this.mediaFormat = mediaFormat; this.mediaFormat = mediaFormat;
this.sampleDescriptionEncryptionBoxes = sampleDescriptionEncryptionBoxes; this.sampleDescriptionEncryptionBoxes = sampleDescriptionEncryptionBoxes;
this.nalUnitLengthFieldLength = nalUnitLengthFieldLength; this.nalUnitLengthFieldLength = nalUnitLengthFieldLength;
this.editListDurations = editListDurations;
this.editListMediaTimes = editListMediaTimes;
} }
} }
library/src/main/java/com/google/android/exoplayer/extractor/mp4/TrackSampleTable.java
...@@ -75,9 +75,11 @@ import com.google.android.exoplayer.util.Util; ...@@ -75,9 +75,11 @@ import com.google.android.exoplayer.util.Util;
* @return Index of the synchronization sample, or {@link #NO_SAMPLE} if none. * @return Index of the synchronization sample, or {@link #NO_SAMPLE} if none.
*/ */
public int getIndexOfEarlierOrEqualSynchronizationSample(long timeUs) { public int getIndexOfEarlierOrEqualSynchronizationSample(long timeUs) {
// Video frame timestamps may not be sorted, so the behavior of this call can be undefined.
// Frames are not reordered past synchronization samples so this works in practice.
int startIndex = Util.binarySearchFloor(timestampsUs, timeUs, true, false); int startIndex = Util.binarySearchFloor(timestampsUs, timeUs, true, false);
for (int i = startIndex; i >= 0; i--) { for (int i = startIndex; i >= 0; i--) {
if (timestampsUs[i] <= timeUs && (flags[i] & C.SAMPLE_FLAG_SYNC) != 0) { if ((flags[i] & C.SAMPLE_FLAG_SYNC) != 0) {
return i; return i;
} }
} }
...@@ -94,7 +96,7 @@ import com.google.android.exoplayer.util.Util; ...@@ -94,7 +96,7 @@ import com.google.android.exoplayer.util.Util;
public int getIndexOfLaterOrEqualSynchronizationSample(long timeUs) { public int getIndexOfLaterOrEqualSynchronizationSample(long timeUs) {
int startIndex = Util.binarySearchCeil(timestampsUs, timeUs, true, false); int startIndex = Util.binarySearchCeil(timestampsUs, timeUs, true, false);
for (int i = startIndex; i < timestampsUs.length; i++) { for (int i = startIndex; i < timestampsUs.length; i++) {
if (timestampsUs[i] >= timeUs && (flags[i] & C.SAMPLE_FLAG_SYNC) != 0) { if ((flags[i] & C.SAMPLE_FLAG_SYNC) != 0) {
return i; return i;
} }
} }
......
library/src/main/java/com/google/android/exoplayer/smoothstreaming/SmoothStreamingChunkSource.java
...@@ -429,8 +429,9 @@ public class SmoothStreamingChunkSource implements ChunkSource, ...@@ -429,8 +429,9 @@ public class SmoothStreamingChunkSource implements ChunkSource,
FragmentedMp4Extractor mp4Extractor = new FragmentedMp4Extractor( FragmentedMp4Extractor mp4Extractor = new FragmentedMp4Extractor(
FragmentedMp4Extractor.WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME FragmentedMp4Extractor.WORKAROUND_EVERY_VIDEO_FRAME_IS_SYNC_FRAME
| FragmentedMp4Extractor.WORKAROUND_IGNORE_TFDT_BOX); | FragmentedMp4Extractor.WORKAROUND_IGNORE_TFDT_BOX);
Track mp4Track = new Track(trackIndex, mp4TrackType, element.timescale, durationUs, mediaFormat, Track mp4Track = new Track(trackIndex, mp4TrackType, element.timescale, C.UNKNOWN_TIME_US,
trackEncryptionBoxes, mp4TrackType == Track.TYPE_vide ? 4 : -1); durationUs, mediaFormat, trackEncryptionBoxes, mp4TrackType == Track.TYPE_vide ? 4 : -1,
null, null);
mp4Extractor.setTrack(mp4Track); mp4Extractor.setTrack(mp4Track);
// Store the format and a wrapper around the extractor. // Store the format and a wrapper around the extractor.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment