Commit 7ea2d75f by Dustin

Refactor Clock logic. Refactor peeking for MP4V and AVC. Moved AVI above MP3.

parent 09485cbe
Showing with 742 additions and 334 deletions
...@@ -98,9 +98,9 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory { ...@@ -98,9 +98,9 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
FileTypes.ADTS, FileTypes.ADTS,
FileTypes.AC3, FileTypes.AC3,
FileTypes.AC4, FileTypes.AC4,
FileTypes.AVI,
FileTypes.MP3, FileTypes.MP3,
FileTypes.JPEG, FileTypes.JPEG,
FileTypes.AVI,
}; };
private static final FlacExtensionLoader FLAC_EXTENSION_LOADER = new FlacExtensionLoader(); private static final FlacExtensionLoader FLAC_EXTENSION_LOADER = new FlacExtensionLoader();
......
package com.google.android.exoplayer2.extractor.avi;
import androidx.annotation.NonNull;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.NalUnitUtil;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.ParsableNalUnitBitArray;
import java.io.IOException;
public class AvcAviTrack extends AviTrack{
private static final int NAL_TYPE_IRD = 5;
private static final int NAL_TYPE_SEI = 6;
private static final int NAL_TYPE_SPS = 7;
private static final int NAL_MASK = 0x1f;
private Format.Builder formatBuilder;
private float pixelWidthHeightRatio = 1f;
private NalUnitUtil.SpsData spsData;
//The frame as a calculated from the picCount
private int picFrame;
private int lastPicCount;
//Largest picFrame, used when we hit an I frame
private int maxPicFrame =-1;
private int maxPicCount;
private int posHalf;
private int negHalf;
AvcAviTrack(int id, @NonNull StreamHeaderBox streamHeaderBox, @NonNull TrackOutput trackOutput,
@NonNull Format.Builder formatBuilder) {
super(id, streamHeaderBox, trackOutput);
this.formatBuilder = formatBuilder;
}
public void setFormatBuilder(Format.Builder formatBuilder) {
this.formatBuilder = formatBuilder;
}
private int seekNal(final ParsableByteArray parsableByteArray) {
final byte[] buffer = parsableByteArray.getData();
for (int i=parsableByteArray.getPosition();i<buffer.length - 5;i++) {
if (buffer[i] == 0 && buffer[i+1] == 0) {
if (buffer[i+2] == 1) {
parsableByteArray.setPosition(i+3);
} else if (buffer[i+2] == 0 && buffer[i+3] == 1) {
parsableByteArray.setPosition(i+4);
} else {
continue;
}
return (parsableByteArray.readUnsignedByte() & NAL_MASK);
}
}
return -1;
}
private void processIdr() {
lastPicCount = 0;
picFrame = maxPicFrame + 1;
}
private void readSps(int size, ExtractorInput input) throws IOException {
final byte[] buffer = new byte[size];
input.readFully(buffer, 0, size, false);
final ParsableByteArray parsableByteArray = new ParsableByteArray(buffer);
int nal;
while ((nal = seekNal(parsableByteArray)) >= 0) {
if (nal == NAL_TYPE_SPS) {
spsData = NalUnitUtil.parseSpsNalUnitPayload(parsableByteArray.getData(), parsableByteArray.getPosition(), parsableByteArray.capacity());
maxPicCount = 1 << (spsData.picOrderCntLsbLength);
posHalf = maxPicCount / 2; //Not sure why pics are 2x
negHalf = -posHalf;
if (spsData.pixelWidthHeightRatio != pixelWidthHeightRatio) {
formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio);
trackOutput.format(formatBuilder.build());
}
Log.d(AviExtractor.TAG, "SPS Frame: maxPicCount=" + maxPicCount);
} else if (nal == NAL_TYPE_IRD) {
processIdr();
}
}
parsableByteArray.setPosition(0);
trackOutput.sampleData(parsableByteArray, parsableByteArray.capacity());
int flags = 0;
if (isKeyFrame()) {
flags |= C.BUFFER_FLAG_KEY_FRAME;
}
trackOutput.sampleMetadata(getUs(frame), flags, parsableByteArray.capacity(), 0, null);
Log.d(AviExtractor.TAG, "SPS Frame: " + (isVideo()? 'V' : 'A') + " us=" + getUs() + " size=" + size + " frame=" + frame + " usFrame=" + getUsFrame());
advance();
}
@Override
int getUsFrame() {
return picFrame;
}
@Override
void seekFrame(int frame) {
super.seekFrame(frame);
this.picFrame = frame;
lastPicCount = 0;
}
int getPicOrderCountLsb(byte[] peek) {
if (peek[3] != 1) {
return -1;
}
final ParsableNalUnitBitArray in = new ParsableNalUnitBitArray(peek, 5, peek.length);
//slide_header()
in.readUnsignedExpGolombCodedInt(); //first_mb_in_slice
in.readUnsignedExpGolombCodedInt(); //slice_type
in.readUnsignedExpGolombCodedInt(); //pic_parameter_set_id
if (spsData.separateColorPlaneFlag) {
in.skipBits(2); //colour_plane_id
}
in.readBits(spsData.frameNumLength); //frame_num
if (!spsData.frameMbsOnlyFlag) {
boolean field_pic_flag = in.readBit(); // field_pic_flag
if (field_pic_flag) {
in.readBit(); // bottom_field_flag
}
}
//We skip IDR in the switch
if (spsData.picOrderCountType == 0) {
int picOrderCountLsb = in.readBits(spsData.picOrderCntLsbLength);
//Log.d("Test", "FrameNum: " + frame + " cnt=" + picOrderCountLsb);
return picOrderCountLsb;
}
return -1;
}
@Override
public boolean newChunk(int tag, int size, ExtractorInput input) throws IOException {
final int peekSize = Math.min(size, 16);
byte[] peek = new byte[peekSize];
input.peekFully(peek, 0, peekSize);
final int nalType = peek[4] & NAL_MASK;
switch (nalType) {
case 1:
case 2:
case 3:
case 4: {
final int picCount = getPicOrderCountLsb(peek);
if (picCount < 0) {
Log.d(AviExtractor.TAG, "Error getting PicOrder");
seekFrame(frame);
}
int delta = picCount - lastPicCount;
if (delta < negHalf) {
delta += maxPicCount;
} else if (delta > posHalf) {
delta -= maxPicCount;
}
picFrame += delta / 2;
lastPicCount = picCount;
if (maxPicFrame < picFrame) {
maxPicFrame = picFrame;
}
break;
}
case NAL_TYPE_IRD:
processIdr();
break;
case NAL_TYPE_SEI:
case NAL_TYPE_SPS:
readSps(size, input);
return true;
}
return super.newChunk(tag, size, input);
}
}
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.util.NalUnitUtil;
import com.google.android.exoplayer2.util.ParsableNalUnitBitArray;
import java.io.IOException;
public class AvcChunkPeeker extends NalChunkPeeker {
private static final int NAL_TYPE_MASK = 0x1f;
private static final int NAL_TYPE_IRD = 5;
private static final int NAL_TYPE_SEI = 6;
private static final int NAL_TYPE_SPS = 7;
private static final int NAL_TYPE_PPS = 8;
private final PicCountClock picCountClock;
private final Format.Builder formatBuilder;
private final TrackOutput trackOutput;
private float pixelWidthHeightRatio = 1f;
private NalUnitUtil.SpsData spsData;
public AvcChunkPeeker(Format.Builder formatBuilder, TrackOutput trackOutput, long usPerChunk) {
super(16);
this.formatBuilder = formatBuilder;
this.trackOutput = trackOutput;
picCountClock = new PicCountClock(usPerChunk);
}
public PicCountClock getPicCountClock() {
return picCountClock;
}
@Override
boolean skip(byte nalType) {
return false;
}
void updatePicCountClock(final int nalTypeOffset) {
final ParsableNalUnitBitArray in = new ParsableNalUnitBitArray(buffer, nalTypeOffset + 1, buffer.length);
//slide_header()
in.readUnsignedExpGolombCodedInt(); //first_mb_in_slice
in.readUnsignedExpGolombCodedInt(); //slice_type
in.readUnsignedExpGolombCodedInt(); //pic_parameter_set_id
if (spsData.separateColorPlaneFlag) {
in.skipBits(2); //colour_plane_id
}
in.readBits(spsData.frameNumLength); //frame_num
if (!spsData.frameMbsOnlyFlag) {
boolean field_pic_flag = in.readBit(); // field_pic_flag
if (field_pic_flag) {
in.readBit(); // bottom_field_flag
}
}
//We skip IDR in the switch
if (spsData.picOrderCountType == 0) {
int picOrderCountLsb = in.readBits(spsData.picOrderCntLsbLength);
//Log.d("Test", "FrameNum: " + frame + " cnt=" + picOrderCountLsb);
picCountClock.setPicCount(picOrderCountLsb);
return;
}
picCountClock.setIndex(picCountClock.getIndex());
}
private int readSps(ExtractorInput input, int nalTypeOffset) throws IOException {
final int spsStart = nalTypeOffset + 1;
nalTypeOffset = seekNextNal(input, spsStart);
spsData = NalUnitUtil.parseSpsNalUnitPayload(buffer, spsStart, pos);
picCountClock.setMaxPicCount(1 << (spsData.picOrderCntLsbLength));
if (spsData.pixelWidthHeightRatio != pixelWidthHeightRatio) {
pixelWidthHeightRatio = spsData.pixelWidthHeightRatio;
formatBuilder.setPixelWidthHeightRatio(pixelWidthHeightRatio);
trackOutput.format(formatBuilder.build());
}
return nalTypeOffset;
}
@Override
void processChunk(ExtractorInput input, int nalTypeOffset) throws IOException {
while (true) {
final int nalType = buffer[nalTypeOffset] & NAL_TYPE_MASK;
switch (nalType) {
case 1:
case 2:
case 3:
case 4:
updatePicCountClock(nalTypeOffset);
return;
case NAL_TYPE_IRD:
picCountClock.syncIndexes();
return;
case NAL_TYPE_SEI:
case NAL_TYPE_PPS: {
nalTypeOffset = seekNextNal(input, nalTypeOffset);
//Usually chunks have other NALs after these, so just continue
break;
}
case NAL_TYPE_SPS:
nalTypeOffset = readSps(input, nalTypeOffset);
//Sometimes video frames lurk after these
break;
default:
return;
}
if (nalTypeOffset < 0) {
return;
}
compact();
}
}
}
...@@ -4,7 +4,7 @@ import java.nio.ByteBuffer; ...@@ -4,7 +4,7 @@ import java.nio.ByteBuffer;
public class AviHeaderBox extends ResidentBox { public class AviHeaderBox extends ResidentBox {
private static final int AVIF_HASINDEX = 0x10; private static final int AVIF_HASINDEX = 0x10;
private static int AVIF_MUSTUSEINDEX = 0x20; private static final int AVIF_MUSTUSEINDEX = 0x20;
static final int AVIH = 'a' | ('v' << 8) | ('i' << 16) | ('h' << 24); static final int AVIH = 'a' | ('v' << 8) | ('i' << 16) | ('h' << 24);
//AVIMAINHEADER //AVIMAINHEADER
...@@ -32,20 +32,29 @@ public class AviHeaderBox extends ResidentBox { ...@@ -32,20 +32,29 @@ public class AviHeaderBox extends ResidentBox {
return byteBuffer.getInt(12); return byteBuffer.getInt(12);
} }
int getFrames() { int getTotalFrames() {
return byteBuffer.getInt(16); return byteBuffer.getInt(16);
} }
//20 = dwInitialFrames
int getSuggestedBufferSize() { // 20 - dwInitialFrames
return byteBuffer.getInt(24); // int getInitialFrames() {
} // return byteBuffer.getInt(20);
// }
int getWidth() { int getStreams() {
return byteBuffer.getInt(28); return byteBuffer.getInt(24);
} }
int getHeight() { // 28 - dwSuggestedBufferSize
return byteBuffer.getInt(32); // int getSuggestedBufferSize() {
} // return byteBuffer.getInt(28);
// }
//
// int getWidth() {
// return byteBuffer.getInt(32);
// }
//
// int getHeight() {
// return byteBuffer.getInt(36);
// }
} }
package com.google.android.exoplayer2.extractor.avi; package com.google.android.exoplayer2.extractor.avi;
import android.util.SparseArray;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import com.google.android.exoplayer2.extractor.SeekMap; import com.google.android.exoplayer2.extractor.SeekMap;
import com.google.android.exoplayer2.extractor.SeekPoint; import com.google.android.exoplayer2.extractor.SeekPoint;
import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Log;
public class AviSeekMap implements SeekMap { public class AviSeekMap implements SeekMap {
final AviTrack videoTrack; final long videoUsPerChunk;
final int videoStreamId;
/** /**
* Number of frames per index * Number of frames per index
* i.e. videoFrameOffsetMap[1] is frame 1 * seekIndexFactor * i.e. videoFrameOffsetMap[1] is frame 1 * seekIndexFactor
*/ */
final int seekIndexFactor; final int seekIndexFactor;
//Map from the Video Frame index to the offset //Seek offsets by streamId, for video, this is the actual offset, for audio, this is the chunkId
final int[] videoFrameOffsetMap; final int[][] seekOffsets;
//Holds a map of video frameIds to audioFrameIds for each audioId //Holds a map of video frameIds to audioFrameIds for each audioId
final SparseArray<int[]> audioIdMap;
final long moviOffset; final long moviOffset;
final long duration; final long duration;
public AviSeekMap(AviTrack videoTrack, int seekIndexFactor, int[] videoFrameOffsetMap, public AviSeekMap(AviTrack videoTrack, UnboundedIntArray[] seekOffsets, int seekIndexFactor, long moviOffset, long duration) {
SparseArray<int[]> audioIdMap, long moviOffset, long duration) { videoUsPerChunk = videoTrack.getClock().usPerChunk;
this.videoTrack = videoTrack; videoStreamId = videoTrack.id;
this.seekIndexFactor = seekIndexFactor; this.seekIndexFactor = seekIndexFactor;
this.videoFrameOffsetMap = videoFrameOffsetMap;
this.audioIdMap = audioIdMap;
this.moviOffset = moviOffset; this.moviOffset = moviOffset;
this.duration = duration; this.duration = duration;
this.seekOffsets = new int[seekOffsets.length][];
for (int i=0;i<seekOffsets.length;i++) {
this.seekOffsets[i] = seekOffsets[i].getArray();
}
} }
@Override @Override
...@@ -41,10 +43,10 @@ public class AviSeekMap implements SeekMap { ...@@ -41,10 +43,10 @@ public class AviSeekMap implements SeekMap {
} }
private int getSeekFrameIndex(long timeUs) { private int getSeekFrameIndex(long timeUs) {
final int reqFrame = (int)(timeUs / videoTrack.usPerSample); final int reqFrame = (int)(timeUs / videoUsPerChunk);
int reqFrameIndex = reqFrame / seekIndexFactor; int reqFrameIndex = reqFrame / seekIndexFactor;
if (reqFrameIndex >= videoFrameOffsetMap.length) { if (reqFrameIndex >= seekOffsets[videoStreamId].length) {
reqFrameIndex = videoFrameOffsetMap.length - 1; reqFrameIndex = seekOffsets[videoStreamId].length - 1;
} }
return reqFrameIndex; return reqFrameIndex;
} }
...@@ -53,23 +55,29 @@ public class AviSeekMap implements SeekMap { ...@@ -53,23 +55,29 @@ public class AviSeekMap implements SeekMap {
@Override @Override
public SeekPoints getSeekPoints(long timeUs) { public SeekPoints getSeekPoints(long timeUs) {
final int seekFrameIndex = getSeekFrameIndex(timeUs); final int seekFrameIndex = getSeekFrameIndex(timeUs);
int offset = videoFrameOffsetMap[seekFrameIndex]; int offset = seekOffsets[videoStreamId][seekFrameIndex];
final long outUs = seekFrameIndex * seekIndexFactor * videoTrack.usPerSample; final long outUs = seekFrameIndex * seekIndexFactor * videoUsPerChunk;
final long position = offset + moviOffset; final long position = offset + moviOffset;
Log.d(AviExtractor.TAG, "SeekPoint: us=" + outUs + " pos=" + position); Log.d(AviExtractor.TAG, "SeekPoint: us=" + outUs + " pos=" + position);
return new SeekPoints(new SeekPoint(outUs, position)); return new SeekPoints(new SeekPoint(outUs, position));
} }
public void setFrames(final long position, final long timeUs, final SparseArray<AviTrack> idTrackMap) { public void setFrames(final long position, final long timeUs, final AviTrack[] aviTracks) {
final int seekFrameIndex = getSeekFrameIndex(timeUs); final int seekFrameIndex = getSeekFrameIndex(timeUs);
videoTrack.seekFrame(seekFrameIndex * seekIndexFactor); for (int i=0;i<aviTracks.length;i++) {
for (int i=0;i<audioIdMap.size();i++) { final AviTrack aviTrack = aviTracks[i];
final int audioId = audioIdMap.keyAt(i); if (aviTrack != null) {
final int[] video2AudioFrameMap = audioIdMap.get(audioId); final LinearClock clock = aviTrack.getClock();
final AviTrack audioTrack = idTrackMap.get(audioId); if (aviTrack.isVideo()) {
audioTrack.frame = video2AudioFrameMap[seekFrameIndex]; //TODO: Although this works, it leads to partial frames being painted
aviTrack.setForceKeyFrame(true);
clock.setIndex(seekFrameIndex * seekIndexFactor);
} else {
final int offset = seekOffsets[i][seekFrameIndex];
clock.setIndex(offset);
}
}
} }
} }
} }
...@@ -18,13 +18,19 @@ public class AviTrack { ...@@ -18,13 +18,19 @@ public class AviTrack {
@NonNull @NonNull
final StreamHeaderBox streamHeaderBox; final StreamHeaderBox streamHeaderBox;
long usPerSample; @NonNull
LinearClock clock;
@Nullable
ChunkPeeker chunkPeeker;
/** /**
* True indicates all frames are key frames (e.g. Audio, MJPEG) * True indicates all frames are key frames (e.g. Audio, MJPEG)
*/ */
boolean allKeyFrames; boolean allKeyFrames;
boolean forceKeyFrame;
@NonNull @NonNull
TrackOutput trackOutput; TrackOutput trackOutput;
...@@ -37,19 +43,24 @@ public class AviTrack { ...@@ -37,19 +43,24 @@ public class AviTrack {
transient int chunkSize; transient int chunkSize;
transient int chunkRemaining; transient int chunkRemaining;
/**
* Current frame in the stream
* This needs to be updated on seek
* TODO: Should be offset from StreamHeaderBox.getStart()
*/
int frame;
AviTrack(int id, @NonNull StreamHeaderBox streamHeaderBox, @NonNull TrackOutput trackOutput) { AviTrack(int id, @NonNull StreamHeaderBox streamHeaderBox, @NonNull TrackOutput trackOutput) {
this.id = id; this.id = id;
this.trackOutput = trackOutput; this.trackOutput = trackOutput;
this.streamHeaderBox = streamHeaderBox; this.streamHeaderBox = streamHeaderBox;
this.usPerSample = streamHeaderBox.getUsPerSample(); clock = new LinearClock(streamHeaderBox.getUsPerSample());
this.allKeyFrames = streamHeaderBox.isAudio() || (MimeTypes.IMAGE_JPEG.equals(streamHeaderBox.getMimeType())); this.allKeyFrames = streamHeaderBox.isAudio() || (MimeTypes.VIDEO_MJPEG.equals(streamHeaderBox.getMimeType()));
}
public LinearClock getClock() {
return clock;
}
public void setClock(LinearClock clock) {
this.clock = clock;
}
public void setChunkPeeker(ChunkPeeker chunkPeeker) {
this.chunkPeeker = chunkPeeker;
} }
public boolean isAllKeyFrames() { public boolean isAllKeyFrames() {
...@@ -60,23 +71,24 @@ public class AviTrack { ...@@ -60,23 +71,24 @@ public class AviTrack {
if (allKeyFrames) { if (allKeyFrames) {
return true; return true;
} }
if (forceKeyFrame) {
forceKeyFrame = false;
return true;
}
if (keyFrames != null) { if (keyFrames != null) {
return Arrays.binarySearch(keyFrames, frame) >= 0; return Arrays.binarySearch(keyFrames, clock.getIndex()) >= 0;
} }
//Hack: Exo needs at least one frame before it starts playback //Hack: Exo needs at least one frame before it starts playback
return frame == 0; //return clock.getIndex() == 0;
return false;
} }
public void setKeyFrames(int[] keyFrames) { public void setForceKeyFrame(boolean v) {
this.keyFrames = keyFrames; forceKeyFrame = v;
}
public long getUs() {
return getUs(getUsFrame());
} }
public long getUs(final int myFrame) { public void setKeyFrames(int[] keyFrames) {
return myFrame * usPerSample; this.keyFrames = keyFrames;
} }
public boolean isVideo() { public boolean isVideo() {
...@@ -87,23 +99,10 @@ public class AviTrack { ...@@ -87,23 +99,10 @@ public class AviTrack {
return streamHeaderBox.isAudio(); return streamHeaderBox.isAudio();
} }
public void advance() {
frame++;
}
/**
* Get the frame number used to calculate the timeUs
* @return
*/
int getUsFrame() {
return frame;
}
void seekFrame(int frame) {
this.frame = frame;
}
public boolean newChunk(int tag, int size, ExtractorInput input) throws IOException { public boolean newChunk(int tag, int size, ExtractorInput input) throws IOException {
if (chunkPeeker != null) {
chunkPeeker.peek(input, size);
}
final int remaining = size - trackOutput.sampleData(input, size, false); final int remaining = size - trackOutput.sampleData(input, size, false);
if (remaining == 0) { if (remaining == 0) {
done(size); done(size);
...@@ -127,8 +126,8 @@ public class AviTrack { ...@@ -127,8 +126,8 @@ public class AviTrack {
void done(final int size) { void done(final int size) {
trackOutput.sampleMetadata( trackOutput.sampleMetadata(
getUs(), (isKeyFrame() ? C.BUFFER_FLAG_KEY_FRAME : 0), size, 0, null); clock.getUs(), (isKeyFrame() ? C.BUFFER_FLAG_KEY_FRAME : 0), size, 0, null);
//Log.d(AviExtractor.TAG, "Frame: " + (isVideo()? 'V' : 'A') + " us=" + getUs() + " size=" + size + " frame=" + frame + " usFrame=" + getUsFrame()); //Log.d(AviExtractor.TAG, "Frame: " + (isVideo()? 'V' : 'A') + " us=" + getUs() + " size=" + size + " frame=" + frame + " usFrame=" + getUsFrame());
advance(); clock.advance();
} }
} }
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import java.io.IOException;
public interface ChunkPeeker {
void peek(ExtractorInput input, final int size) throws IOException;
}
package com.google.android.exoplayer2.extractor.avi;
public class LinearClock {
long usPerChunk;
int index;
public LinearClock(long usPerChunk) {
this.usPerChunk = usPerChunk;
}
public int getIndex() {
return index;
}
public void setIndex(int index) {
this.index = index;
}
public void advance() {
index++;
}
public long getUs() {
return index * usPerChunk;
}
}
package com.google.android.exoplayer2.extractor.avi; package com.google.android.exoplayer2.extractor.avi;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.extractor.ExtractorInput; import com.google.android.exoplayer2.extractor.ExtractorInput;
...@@ -9,22 +8,35 @@ import com.google.android.exoplayer2.extractor.TrackOutput; ...@@ -9,22 +8,35 @@ import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.util.ParsableNalUnitBitArray; import com.google.android.exoplayer2.util.ParsableNalUnitBitArray;
import java.io.IOException; import java.io.IOException;
public class Mp4vAviTrack extends AviTrack { public class Mp4vChunkPeeker extends NalChunkPeeker {
private static final byte SEQUENCE_START_CODE = (byte)0xb0; @VisibleForTesting
private static final int LAYER_START_CODE = 0x20; static final byte SEQUENCE_START_CODE = (byte)0xb0;
@VisibleForTesting
static final int LAYER_START_CODE = 0x20;
private static final float[] ASPECT_RATIO = {0f, 1f, 12f/11f, 10f/11f, 16f/11f, 40f/33f}; private static final float[] ASPECT_RATIO = {0f, 1f, 12f/11f, 10f/11f, 16f/11f, 40f/33f};
private static final int Extended_PAR = 0xf; @VisibleForTesting
static final int Extended_PAR = 0xf;
private final Format.Builder formatBuilder; private final Format.Builder formatBuilder;
private final TrackOutput trackOutput;
@VisibleForTesting()
float pixelWidthHeightRatio = 1f; float pixelWidthHeightRatio = 1f;
Mp4vAviTrack(int id, @NonNull StreamHeaderBox streamHeaderBox, @NonNull TrackOutput trackOutput, public Mp4vChunkPeeker(@NonNull Format.Builder formatBuilder, @NonNull TrackOutput trackOutput) {
@NonNull Format.Builder formatBuilder) { super(5);
super(id, streamHeaderBox, trackOutput);
this.formatBuilder = formatBuilder; this.formatBuilder = formatBuilder;
this.trackOutput = trackOutput;
}
@Override
boolean skip(byte nalType) {
return nalType != SEQUENCE_START_CODE;
} }
@VisibleForTesting @VisibleForTesting
void processLayerStart(@NonNull final ParsableNalUnitBitArray in) { void processLayerStart(int nalTypeOffset) {
@NonNull final ParsableNalUnitBitArray in = new ParsableNalUnitBitArray(buffer, nalTypeOffset + 1, pos);
in.skipBit(); // random_accessible_vol in.skipBit(); // random_accessible_vol
in.skipBits(8); // video_object_type_indication in.skipBits(8); // video_object_type_indication
boolean is_object_layer_identifier = in.readBit(); boolean is_object_layer_identifier = in.readBit();
...@@ -46,36 +58,19 @@ public class Mp4vAviTrack extends AviTrack { ...@@ -46,36 +58,19 @@ public class Mp4vAviTrack extends AviTrack {
} }
} }
@VisibleForTesting
@Nullable
static ParsableNalUnitBitArray findLayerStart(ExtractorInput input, final int peekSize)
throws IOException {
byte[] peek = new byte[peekSize];
input.peekFully(peek, 0, peekSize);
for (int i = 4;i<peek.length - 4;i++) {
if (peek[i] == 0 && peek[i+1] == 0 && peek[i+2] == 1 && (peek[i+3] & 0xf0) == LAYER_START_CODE) {
return new ParsableNalUnitBitArray(peek, i+4, peek.length);
}
}
return null;
}
@VisibleForTesting
static boolean isSequenceStart(ExtractorInput input) throws IOException {
final byte[] peek = new byte[4];
input.peekFully(peek, 0, peek.length);
return peek[0] == 0 && peek[1] == 0 && peek[2] == 1 && peek[3] == SEQUENCE_START_CODE;
}
@Override @Override
public boolean newChunk(int tag, int size, ExtractorInput input) throws IOException { void processChunk(ExtractorInput input, int nalTypeOffset) throws IOException {
if (isSequenceStart(input)) { while (true) {
// -4 because isSequenceStart peeks 4 if ((buffer[nalTypeOffset] & 0xf0) == LAYER_START_CODE) {
final ParsableNalUnitBitArray layerStart = findLayerStart(input, Math.min(size - 4, 128)); seekNextNal(input, nalTypeOffset);
if (layerStart != null) { processLayerStart(nalTypeOffset);
processLayerStart(layerStart); break;
}
nalTypeOffset = seekNextNal(input, nalTypeOffset);
if (nalTypeOffset < 0) {
break;
} }
compact();
} }
return super.newChunk(tag, size, input);
} }
} }
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import java.io.IOException;
import java.util.Arrays;
public abstract class NalChunkPeeker implements ChunkPeeker {
private static final int SEEK_PEEK_SIZE = 256;
private final int peekSize;
private transient int remaining;
transient byte[] buffer;
transient int pos;
abstract void processChunk(ExtractorInput input, int nalTypeOffset) throws IOException;
/**
*
* @return NAL offset from pos
*/
private int getNalTypeOffset() {
if (buffer[pos] == 0 && buffer[pos+1] == 0) {
if (buffer[pos+2] == 1) {
return 3;
} else if (buffer[pos+2] == 0 && buffer[pos+3] == 1) {
return 4;
}
}
return -1;
}
/**
* Look for the next NAL in buffer, incrementing pos
* @return offset of the nal from the pos
*/
private int seekNal() {
int nalOffset;
while ((nalOffset = getNalTypeOffset()) < 0 && pos < buffer.length - 5) {
pos++;
}
return nalOffset;
}
/**
* Removes everything before the pos
*/
void compact() {
//Compress down to the last NAL
final byte[] newBuffer = new byte[buffer.length - pos];
System.arraycopy(buffer, pos, newBuffer, 0, newBuffer.length);
buffer = newBuffer;
pos = 0;
}
/**
* @param peekSize number of bytes to append
*/
void append(final ExtractorInput input, final int peekSize) throws IOException {
int oldLength = buffer.length;
buffer = Arrays.copyOf(buffer, oldLength + peekSize);
input.peekFully(buffer, oldLength, peekSize);
remaining -= peekSize;
}
/**
*
* @return NAL offset from pos, -1 if end of input
*/
int seekNextNal(final ExtractorInput input, int skip) throws IOException {
pos += skip;
while (pos + 5 < buffer.length || remaining > 0) {
if (buffer.length - pos < SEEK_PEEK_SIZE && remaining > 0) {
append(input, Math.min(SEEK_PEEK_SIZE, remaining));
}
final int nalOffset = seekNal();
if (nalOffset > 0) {
return nalOffset;
}
}
pos = buffer.length;
return -1;
}
public NalChunkPeeker(int peakSize) {
if (peakSize < 5) {
throw new IllegalArgumentException("Peak size must at least be 5");
}
this.peekSize = peakSize;
}
abstract boolean skip(byte nalType);
public void peek(ExtractorInput input, final int size) throws IOException {
buffer = new byte[peekSize];
if (!input.peekFully(buffer, 0, peekSize, true)) {
return;
}
pos = 0;
int nalTypeOffset = getNalTypeOffset();
if (nalTypeOffset < 0 || skip(buffer[nalTypeOffset])) {
input.resetPeekPosition();
return;
}
remaining = size - peekSize;
processChunk(input, nalTypeOffset);
input.resetPeekPosition();
}
// @VisibleForTesting(otherwise = VisibleForTesting.NONE)
// void setBuffer(byte[] buffer) {
// this.buffer = buffer;
// }
}
package com.google.android.exoplayer2.extractor.avi;
/**
* Properly calculates the frame time for H264 frames using PicCount
*/
public class PicCountClock extends LinearClock {
//The frame as a calculated from the picCount
private int picIndex;
private int lastPicCount;
//Largest picFrame, used when we hit an I frame
private int maxPicIndex =-1;
private int maxPicCount;
private int posHalf;
private int negHalf;
public PicCountClock(long usPerFrame) {
super(usPerFrame);
}
public void setMaxPicCount(int maxPicCount) {
this.maxPicCount = maxPicCount;
posHalf = maxPicCount / 2; //Not sure why pics are 2x
negHalf = -posHalf;
}
/**
* Done on seek. May cause sync issues if frame picCount != 0 (I frames are always 0)
* @param index
*/
@Override
public void setIndex(int index) {
super.setIndex(index);
syncIndexes();
}
public void setPicCount(int picCount) {
int delta = picCount - lastPicCount;
if (delta < negHalf) {
delta += maxPicCount;
} else if (delta > posHalf) {
delta -= maxPicCount;
}
picIndex += delta / 2;
lastPicCount = picCount;
if (maxPicIndex < picIndex) {
maxPicIndex = picIndex;
}
}
/**
* Handle key frame
*/
public void syncIndexes() {
lastPicCount = 0;
maxPicIndex = picIndex = getIndex();
}
@Override
public long getUs() {
return picIndex * usPerChunk;
}
}
...@@ -36,6 +36,7 @@ public class StreamHeaderBox extends ResidentBox { ...@@ -36,6 +36,7 @@ public class StreamHeaderBox extends ResidentBox {
STREAM_MAP.put('x' | ('v' << 8) | ('i' << 16) | ('d' << 24), mimeType); STREAM_MAP.put('x' | ('v' << 8) | ('i' << 16) | ('d' << 24), mimeType);
STREAM_MAP.put(XVID, mimeType); STREAM_MAP.put(XVID, mimeType);
STREAM_MAP.put('D' | ('X' << 8) | ('5' << 16) | ('0' << 24), mimeType); STREAM_MAP.put('D' | ('X' << 8) | ('5' << 16) | ('0' << 24), mimeType);
STREAM_MAP.put('d' | ('i' << 8) | ('v' << 16) | ('x' << 24), mimeType);
STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.VIDEO_MJPEG); STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.VIDEO_MJPEG);
} }
......
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.testutil.FakeExtractorInput;
import com.google.android.exoplayer2.testutil.FakeExtractorOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.junit.Assert;
import org.junit.Test;
public class AviExtractorTest {
@Test
public void init_givenFakeExtractorOutput() {
AviExtractor aviExtractor = new AviExtractor();
FakeExtractorOutput output = new FakeExtractorOutput();
aviExtractor.init(output);
Assert.assertEquals(AviExtractor.STATE_READ_TRACKS, aviExtractor.state);
Assert.assertEquals(output, aviExtractor.output);
}
private boolean sniff(ByteBuffer byteBuffer) {
AviExtractor aviExtractor = new AviExtractor();
FakeExtractorInput input = new FakeExtractorInput.Builder()
.setData(byteBuffer.array()).build();
try {
return aviExtractor.sniff(input);
} catch (IOException e) {
Assert.fail(e.getMessage());
return false;
}
}
@Test
public void peek_givenTooFewByte() {
Assert.assertFalse(sniff(AviExtractor.allocate(AviExtractor.PEEK_BYTES - 1)));
}
@Test
public void peek_givenAllZero() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
Assert.assertFalse(sniff(byteBuffer));
}
@Test
public void peek_givenOnlyRiff() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
byteBuffer.putInt(AviExtractor.RIFF);
Assert.assertFalse(sniff(byteBuffer));
}
@Test
public void peek_givenOnlyRiffAvi_() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
byteBuffer.putInt(AviExtractor.RIFF);
byteBuffer.putInt(128);
byteBuffer.putInt(AviExtractor.AVI_);
Assert.assertFalse(sniff(byteBuffer));
}
@Test
public void peek_givenOnlyRiffAvi_List() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
byteBuffer.putInt(AviExtractor.RIFF);
byteBuffer.putInt(128);
byteBuffer.putInt(AviExtractor.AVI_);
byteBuffer.putInt(ListBox.LIST);
Assert.assertFalse(sniff(byteBuffer));
}
@Test
public void peek_givenOnlyRiffAvi_ListHdrl() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
byteBuffer.putInt(AviExtractor.RIFF);
byteBuffer.putInt(128);
byteBuffer.putInt(AviExtractor.AVI_);
byteBuffer.putInt(ListBox.LIST);
byteBuffer.putInt(64);
byteBuffer.putInt(ListBox.TYPE_HDRL);
Assert.assertFalse(sniff(byteBuffer));
}
@Test
public void peek_givenOnlyRiffAvi_ListHdrlAvih() {
ByteBuffer byteBuffer = AviExtractor.allocate(AviExtractor.PEEK_BYTES);
byteBuffer.putInt(AviExtractor.RIFF);
byteBuffer.putInt(128);
byteBuffer.putInt(AviExtractor.AVI_);
byteBuffer.putInt(ListBox.LIST);
byteBuffer.putInt(64);
byteBuffer.putInt(ListBox.TYPE_HDRL);
byteBuffer.putInt(AviHeaderBox.AVIH);
Assert.assertTrue(sniff(byteBuffer));
}
@Test
public void toString_givenKnownString() {
final int riff = 'R' | ('I' << 8) | ('F' << 16) | ('F' << 24);
Assert.assertEquals("RIFF", AviExtractor.toString(riff));
}
}
package com.google.android.exoplayer2.extractor.avi;
import java.nio.BufferOverflowException;
public class BitBuffer {
private long work;
int bits;
public void push(boolean b) {
grow(1);
if (b) {
work |= 1L;
}
}
void grow(int bits) {
if (this.bits + bits > 64) {
throw new BufferOverflowException();
}
this.bits += bits;
work <<= bits;
}
public void push(int bits, int value) {
int mask = (1 << bits) - 1;
if ((value & mask) != value) {
throw new IllegalArgumentException("Expected only " + bits + " bits, got " + value);
}
grow(bits);
work |= (value & 0xffffffffL);
}
public byte[] getBytes() {
//Byte align
grow(8 - bits % 8);
final int count = bits / 8;
final byte[] bytes = new byte[count];
for (int i=count -1; i >= 0;i--) {
bytes[i] = (byte)(work & 0xff);
work >>=8;
}
work = 0L;
bits = 0;
return bytes;
}
}
...@@ -50,4 +50,12 @@ public class DataHelper { ...@@ -50,4 +50,12 @@ public class DataHelper {
bytes = Arrays.copyOf(bytes, bytes.length + 1); bytes = Arrays.copyOf(bytes, bytes.length + 1);
return new StreamNameBox(StreamNameBox.STRN, bytes.length, ByteBuffer.wrap(bytes)); return new StreamNameBox(StreamNameBox.STRN, bytes.length, ByteBuffer.wrap(bytes));
} }
public static ByteBuffer appendNal(final ByteBuffer byteBuffer, byte nalType) {
byteBuffer.put((byte)0);
byteBuffer.put((byte)0);
byteBuffer.put((byte) 1);
byteBuffer.put(nalType);
return byteBuffer;
}
} }
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import java.io.IOException;
public class MockNalChunkPeeker extends NalChunkPeeker {
private boolean skip;
public MockNalChunkPeeker(int peakSize, boolean skip) {
super(peakSize);
this.skip = skip;
}
@Override
void processChunk(ExtractorInput input, int nalTypeOffset) throws IOException {
}
@Override
boolean skip(byte nalType) {
return skip;
}
}
package com.google.android.exoplayer2.extractor.avi;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.testutil.FakeExtractorInput;
import com.google.android.exoplayer2.testutil.FakeTrackOutput;
import com.google.android.exoplayer2.util.ParsableNalUnitBitArray;
import java.io.IOException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class Mp4vAviTrackTest {
@Test
public void isSequenceStart_givenSequence() throws IOException {
final FakeExtractorInput input = DataHelper.getInput("mp4v_sequence.dump");
Assert.assertTrue(Mp4vAviTrack.isSequenceStart(input));
}
@Test
public void findLayerStart_givenSequence() throws IOException {
final FakeExtractorInput input = DataHelper.getInput("mp4v_sequence.dump");
final ParsableNalUnitBitArray bitArray = Mp4vAviTrack.findLayerStart(input,
(int)input.getLength());
//Offset 0x12
Assert.assertEquals(8, bitArray.readBits(8));
}
@Test
public void findLayerStart_givenAllZeros() throws IOException {
final FakeExtractorInput fakeExtractorInput = new FakeExtractorInput.Builder().
setData(new byte[128]).build();
Assert.assertNull(Mp4vAviTrack.findLayerStart(fakeExtractorInput, 128));
}
@Test
public void pixelWidthHeightRatio_givenSequence() throws IOException {
final FakeTrackOutput fakeTrackOutput = new FakeTrackOutput(false);
final Format.Builder formatBuilder = new Format.Builder();
final Mp4vAviTrack mp4vAviTrack = new Mp4vAviTrack(0, DataHelper.getVidsStreamHeader(),
fakeTrackOutput, formatBuilder);
final FakeExtractorInput input = DataHelper.getInput("mp4v_sequence.dump");
mp4vAviTrack.newChunk(0, (int)input.getLength(), input);
// final ParsableNalUnitBitArray bitArray = Mp4vAviTrack.findLayerStart(input,
// (int)input.getLength());
// mp4vAviTrack.processLayerStart(bitArray);
Assert.assertEquals(mp4vAviTrack.pixelWidthHeightRatio, 1.2121212, 0.01);
}
}
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.testutil.FakeExtractorInput;
import com.google.android.exoplayer2.testutil.FakeTrackOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.junit.Assert;
import org.junit.Test;
public class Mp4vChunkPeekerTest {
private ByteBuffer makeSequence() {
return DataHelper.appendNal(AviExtractor.allocate(32),Mp4vChunkPeeker.SEQUENCE_START_CODE);
}
@Test
public void peek_givenNoSequence() throws IOException {
ByteBuffer byteBuffer = makeSequence();
final FakeTrackOutput fakeTrackOutput = new FakeTrackOutput(false);
final Format.Builder formatBuilder = new Format.Builder();
final FakeExtractorInput input = new FakeExtractorInput.Builder().setData(byteBuffer.array())
.build();
final Mp4vChunkPeeker mp4vChunkPeeker = new Mp4vChunkPeeker(formatBuilder, fakeTrackOutput);
mp4vChunkPeeker.peek(input, (int) input.getLength());
Assert.assertEquals(1f, mp4vChunkPeeker.pixelWidthHeightRatio, 0.01);
}
@Test
public void peek_givenAspectRatio() throws IOException {
final FakeTrackOutput fakeTrackOutput = new FakeTrackOutput(false);
final Format.Builder formatBuilder = new Format.Builder();
final Mp4vChunkPeeker mp4vChunkPeeker = new Mp4vChunkPeeker(formatBuilder, fakeTrackOutput);
final FakeExtractorInput input = DataHelper.getInput("mp4v_sequence.dump");
mp4vChunkPeeker.peek(input, (int) input.getLength());
Assert.assertEquals(1.2121212, mp4vChunkPeeker.pixelWidthHeightRatio, 0.01);
}
@Test
public void peek_givenCustomAspectRatio() throws IOException {
ByteBuffer byteBuffer = makeSequence();
byteBuffer.putInt(0x5555);
DataHelper.appendNal(byteBuffer, (byte)Mp4vChunkPeeker.LAYER_START_CODE);
BitBuffer bitBuffer = new BitBuffer();
bitBuffer.push(false); //random_accessible_vol
bitBuffer.push(8, 8); //video_object_type_indication
bitBuffer.push(true); // is_object_layer_identifier
bitBuffer.push(7, 7); // video_object_layer_verid, video_object_layer_priority
bitBuffer.push(4, Mp4vChunkPeeker.Extended_PAR);
bitBuffer.push(8, 16);
bitBuffer.push(8, 9);
final byte bytes[] = bitBuffer.getBytes();
byteBuffer.put(bytes);
final FakeTrackOutput fakeTrackOutput = new FakeTrackOutput(false);
final Format.Builder formatBuilder = new Format.Builder();
final FakeExtractorInput input = new FakeExtractorInput.Builder().setData(byteBuffer.array())
.build();
final Mp4vChunkPeeker mp4vChunkPeeker = new Mp4vChunkPeeker(formatBuilder, fakeTrackOutput);
mp4vChunkPeeker.peek(input, (int) input.getLength());
Assert.assertEquals(16f/9f, mp4vChunkPeeker.pixelWidthHeightRatio, 0.01);
}
}
\ No newline at end of file
package com.google.android.exoplayer2.extractor.avi;
import com.google.android.exoplayer2.testutil.FakeExtractorInput;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.junit.Assert;
import org.junit.Test;
public class NalChunkPeekerTest {
@Test
public void construct_givenTooSmallPeekSize() {
try {
new MockNalChunkPeeker(4, false);
Assert.fail();
} catch (IllegalArgumentException e) {
//Intentionally blank
}
}
@Test
public void peek_givenNoData() {
final FakeExtractorInput input = new FakeExtractorInput.Builder().build();
final MockNalChunkPeeker peeker = new MockNalChunkPeeker(5, false);
try {
peeker.peek(input, 10);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
@Test
public void peek_givenNoNal() {
final FakeExtractorInput input = new FakeExtractorInput.Builder().setData(new byte[10]).build();
final MockNalChunkPeeker peeker = new MockNalChunkPeeker(5, false);
try {
peeker.peek(input, 10);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
@Test
public void peek_givenAlwaysSkip() {
final ByteBuffer byteBuffer = AviExtractor.allocate(10);
DataHelper.appendNal(byteBuffer, (byte)32);
final FakeExtractorInput input = new FakeExtractorInput.Builder().setData(byteBuffer.array()).build();
final MockNalChunkPeeker peeker = new MockNalChunkPeeker(5, true);
try {
peeker.peek(input, 10);
Assert.assertEquals(0, input.getPeekPosition());
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment