Skip to content

SDK / exoplayer

Go to a project
Commit f5796b2d by hschlueter Committed by Ian Baker
Options

Use background thread for FrameEditor's OpenGL calls. 

If an OpenGL call blocks because the encoder's input surface is full,
this will now block the background thread while the main thread can
continue querying encoder output and free up encoder capacity until
it accepts more input unblocking the background thread.

PiperOrigin-RevId: 433283287
parent ff6e641f
Expand all
Inline Side-by-side
Showing with 73 additions and 80 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java
...@@ -44,7 +44,7 @@ import org.junit.Test; ...@@ -44,7 +44,7 @@ import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
/** /**
* Pixel test for frame processing via {@link FrameEditor#processData()}. * Pixel test for frame processing via {@link FrameEditor}.
* *
* <p>Expected images are taken from an emulator, so tests on different emulators or physical * <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link * devices may fail. To test on other devices, please increase the {@link
...@@ -61,8 +61,11 @@ public final class FrameEditorDataProcessingTest { ...@@ -61,8 +61,11 @@ public final class FrameEditorDataProcessingTest {
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
/** Timeout for dequeueing buffers from the codec, in microseconds. */ /** Timeout for dequeueing buffers from the codec, in microseconds. */
private static final int DEQUEUE_TIMEOUT_US = 5_000_000; private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
/** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */ /**
private static final int SURFACE_WAIT_MS = 1000; * Time to wait for the decoded frame to populate the frame editor's input surface and the frame
* editor to finish processing the frame, in milliseconds.
*/
private static final int FRAME_PROCESSING_WAIT_MS = 1000;
/** The ratio of width over height, for each pixel in a frame. */ /** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
...@@ -84,10 +87,7 @@ public final class FrameEditorDataProcessingTest { ...@@ -84,10 +87,7 @@ public final class FrameEditorDataProcessingTest {
setUpAndPrepareFirstFrame(identityMatrix); setUpAndPrepareFirstFrame(identityMatrix);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData(); Bitmap actualBitmap = processFirstFrameAndEnd();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
editorOutputImage.close();
// TODO(b/207848601): switch to using proper tooling for testing against golden data. // TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
...@@ -107,10 +107,7 @@ public final class FrameEditorDataProcessingTest { ...@@ -107,10 +107,7 @@ public final class FrameEditorDataProcessingTest {
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData(); Bitmap actualBitmap = processFirstFrameAndEnd();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
editorOutputImage.close();
// TODO(b/207848601): switch to using proper tooling for testing against golden data. // TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
...@@ -130,10 +127,7 @@ public final class FrameEditorDataProcessingTest { ...@@ -130,10 +127,7 @@ public final class FrameEditorDataProcessingTest {
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData(); Bitmap actualBitmap = processFirstFrameAndEnd();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
editorOutputImage.close();
// TODO(b/207848601): switch to using proper tooling for testing against golden data. // TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
...@@ -155,10 +149,7 @@ public final class FrameEditorDataProcessingTest { ...@@ -155,10 +149,7 @@ public final class FrameEditorDataProcessingTest {
setUpAndPrepareFirstFrame(rotate90Matrix); setUpAndPrepareFirstFrame(rotate90Matrix);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData(); Bitmap actualBitmap = processFirstFrameAndEnd();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
editorOutputImage.close();
// TODO(b/207848601): switch to using proper tooling for testing against golden data. // TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
...@@ -205,7 +196,7 @@ public final class FrameEditorDataProcessingTest { ...@@ -205,7 +196,7 @@ public final class FrameEditorDataProcessingTest {
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure( mediaCodec.configure(
mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0); mediaFormat, frameEditor.createInputSurface(), /* crypto= */ null, /* flags= */ 0);
mediaCodec.start(); mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
...@@ -237,10 +228,6 @@ public final class FrameEditorDataProcessingTest { ...@@ -237,10 +228,6 @@ public final class FrameEditorDataProcessingTest {
} while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
|| outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED); || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true); mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
// Sleep to give time for the surface texture to be populated.
Thread.sleep(SURFACE_WAIT_MS);
assertThat(frameEditor.canProcessData()).isTrue();
} finally { } finally {
mediaExtractor.release(); mediaExtractor.release();
if (mediaCodec != null) { if (mediaCodec != null) {
...@@ -248,4 +235,16 @@ public final class FrameEditorDataProcessingTest { ...@@ -248,4 +235,16 @@ public final class FrameEditorDataProcessingTest {
} }
} }
} }
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException {
checkNotNull(frameEditor).signalEndOfInputStream();
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameEditor.isEnded()).isTrue();
frameEditor.getAndRethrowBackgroundExceptions();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
editorOutputImage.close();
return actualBitmap;
}
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -48,7 +48,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -48,7 +48,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final Codec encoder; private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer; private final DecoderInputBuffer encoderOutputBuffer;
private boolean waitingForFrameEditorInput; private boolean signaledEndOfStreamToEncoder;
public VideoTranscodingSamplePipeline( public VideoTranscodingSamplePipeline(
Context context, Context context,
...@@ -177,7 +177,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -177,7 +177,7 @@ import org.checkerframework.dataflow.qual.Pure;
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(
inputFormat, inputFormat,
frameEditor == null ? encoder.getInputSurface() : frameEditor.getInputSurface()); frameEditor == null ? encoder.getInputSurface() : frameEditor.createInputSurface());
} }
@Override @Override
...@@ -193,18 +193,39 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -193,18 +193,39 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() throws TransformationException {
if (hasProcessedAllInputData()) { if (frameEditor != null) {
frameEditor.getAndRethrowBackgroundExceptions();
if (frameEditor.isEnded()) {
if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
}
return false;
}
}
if (decoder.isEnded()) {
return false; return false;
} }
boolean canProcessMoreDataImmediately = false;
if (SDK_INT >= 29 if (SDK_INT >= 29
&& !(("samsung".equals(Util.MANUFACTURER) || "OnePlus".equals(Util.MANUFACTURER)) && !(("samsung".equals(Util.MANUFACTURER) || "OnePlus".equals(Util.MANUFACTURER))
&& SDK_INT < 31)) { && SDK_INT < 31)) {
// TODO(b/213455700): Fix Samsung and OnePlus devices filling the decoder in processDataV29(). // TODO(b/213455700): Fix Samsung and OnePlus devices filling the decoder in processDataV29().
return processDataV29(); processDataV29();
} else { } else {
return processDataDefault(); canProcessMoreDataImmediately = processDataDefault();
}
if (decoder.isEnded()) {
if (frameEditor != null) {
frameEditor.signalEndOfInputStream();
} else {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
return false;
}
} }
return canProcessMoreDataImmediately;
} }
/** /**
...@@ -221,54 +242,22 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -221,54 +242,22 @@ import org.checkerframework.dataflow.qual.Pure;
* Transformer}, using this method requires API level 29 or higher. * Transformer}, using this method requires API level 29 or higher.
*/ */
@RequiresApi(29) @RequiresApi(29)
private boolean processDataV29() throws TransformationException { private void processDataV29() throws TransformationException {
if (frameEditor != null) { while (maybeProcessDecoderOutput()) {}
// Processes as many frames as possible. FrameEditor's output surface will block when it's
// full, so there will be no frame drop and the surface will not grow out of bound.
while (frameEditor.canProcessData()) {
frameEditor.processData();
}
}
while (decoder.getOutputBufferInfo() != null) {
if (frameEditor != null) {
frameEditor.registerInputFrame();
}
decoder.releaseOutputBuffer(/* render= */ true);
}
if (decoder.isEnded()) {
signalEndOfInputStream();
}
return frameEditor != null && frameEditor.canProcessData();
} }
/** Processes input data. */ /**
* Processes at most one input frame and returns whether a frame was processed.
*
* <p>Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link
* FrameEditor} has finished processing the previous frame.
*/
private boolean processDataDefault() throws TransformationException { private boolean processDataDefault() throws TransformationException {
if (frameEditor != null) { // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
if (frameEditor.canProcessData()) { if (frameEditor != null && frameEditor.hasPendingFrames()) {
waitingForFrameEditorInput = false;
frameEditor.processData();
return true;
}
if (waitingForFrameEditorInput) {
return false;
}
}
boolean decoderHasOutputBuffer = decoder.getOutputBufferInfo() != null;
if (decoderHasOutputBuffer) {
if (frameEditor != null) {
frameEditor.registerInputFrame();
waitingForFrameEditorInput = true;
}
decoder.releaseOutputBuffer(/* render= */ true);
}
if (decoder.isEnded()) {
signalEndOfInputStream();
return false; return false;
} }
return decoderHasOutputBuffer && !waitingForFrameEditorInput; return maybeProcessDecoderOutput();
} }
@Override @Override
...@@ -332,16 +321,21 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -332,16 +321,21 @@ import org.checkerframework.dataflow.qual.Pure;
.build(); .build();
} }
private boolean hasProcessedAllInputData() { /**
return decoder.isEnded() && (frameEditor == null || frameEditor.isEnded()); * Feeds at most one decoder output frame to the next step of the pipeline.
} *
* @return Whether a frame was processed.
* @throws TransformationException If a problem occurs while processing the frame.
*/
private boolean maybeProcessDecoderOutput() throws TransformationException {
if (decoder.getOutputBufferInfo() == null) {
return false;
}
private void signalEndOfInputStream() throws TransformationException {
if (frameEditor != null) { if (frameEditor != null) {
frameEditor.signalEndOfInputStream(); frameEditor.registerInputFrame();
}
if (frameEditor == null || frameEditor.isEnded()) {
encoder.signalEndOfInputStream();
} }
decoder.releaseOutputBuffer(/* render= */ true);
return true;
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment