Skip to content

SDK / exoplayer

Go to a project
Commit 2a363ac3 by huangdarwin Committed by Ian Baker
Options

Transformer: Always use FrameProcessorChain when decoding. 

This allows us to bypass many device-specific issues, that only occur when
decoding directly to an encoder surface, without OpenGL. This also allows us
to maintain fewer code branches, which require additional testing to verify
correctness.

PiperOrigin-RevId: 437003138
parent e8b0971f
Inline Side-by-side
Showing with 31 additions and 136 deletions
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
......@@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import androidx.media3.common.MimeTypes;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
......@@ -32,38 +31,9 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class TransformerEndToEndTest {
private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm";
private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4";
@Test
public void videoTranscoding_completesWithConsistentFrameCount() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
FrameCountingMuxer.Factory muxerFactory =
new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory());
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build())
.setMuxerFactory(muxerFactory)
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
// Result of the following command:
// ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm
int expectedFrameCount = 82;
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(
/* testId= */ "videoTranscoding_completesWithConsistentFrameCount",
VP9_VIDEO_URI_STRING);
FrameCountingMuxer frameCountingMuxer =
checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated());
assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount);
}
@Test
public void videoEditing_completesWithConsistentFrameCount() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
FrameCountingMuxer.Factory muxerFactory =
......
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
......@@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
private int inputWidth;
private int inputHeight;
private int outputHeight;
private int outputRotationDegrees;
private @MonotonicNonNull Matrix transformationMatrix;
......@@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputWidth = C.LENGTH_UNSET;
inputHeight = C.LENGTH_UNSET;
outputHeight = C.LENGTH_UNSET;
outputRotationDegrees = C.LENGTH_UNSET;
}
......@@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return outputRotationDegrees;
}
/**
* Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame.
*
* <p>The {@code PresentationFrameProcessor} should only be used if this returns true.
*
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/
public boolean shouldProcess() {
checkStateNotNull(transformationMatrix);
return inputHeight != outputHeight || !transformationMatrix.isIdentity();
}
@Override
public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
......@@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
displayHeight = requestedHeight;
}
int outputWidth;
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// frame before encoding, so the encoded frame's width >= height, and set
// outputRotationDegrees to ensure the frame is displayed in the correct orientation.
if (displayHeight > displayWidth) {
outputRotationDegrees = 90;
outputWidth = displayHeight;
outputHeight = displayWidth;
// TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in
// a later GlFrameProcessor.
// TODO(b/201293185): After fragment shader transformations are implemented, put
// postRotate in a later GlFrameProcessor.
transformationMatrix.postRotate(outputRotationDegrees);
return new Size(displayHeight, displayWidth);
} else {
outputRotationDegrees = 0;
outputWidth = displayWidth;
outputHeight = displayHeight;
return new Size(displayWidth, displayHeight);
}
return new Size(outputWidth, outputHeight);
}
@Override
......
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
......@@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputHeight = C.LENGTH_UNSET;
}
/**
* Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame.
*
* <p>The ScaleToFitFrameProcessor should only be used if this returns true.
*
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/
public boolean shouldProcess() {
checkStateNotNull(adjustedTransformationMatrix);
return !transformationMatrix.isIdentity();
}
@Override
public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
......@@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder;
@Nullable private final FrameProcessorChain frameProcessorChain;
private final FrameProcessorChain frameProcessorChain;
private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer;
......@@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder(context)
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
......@@ -109,36 +110,24 @@ import org.checkerframework.dataflow.qual.Pure;
requestedEncoderFormat,
encoderSupportedFormat));
if (transformationRequest.enableHdrEditing
|| inputFormat.height != encoderSupportedFormat.height
|| inputFormat.width != encoderSupportedFormat.width
|| scaleToFitFrameProcessor.shouldProcess()
|| presentationFrameProcessor.shouldProcess()
|| shouldAlwaysUseFrameProcessorChain()) {
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
// has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set(
frameProcessorSizes.size() - 1,
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
frameProcessorChain =
FrameProcessorChain.create(
context,
inputFormat.pixelWidthHeightRatio,
frameProcessors,
frameProcessorSizes,
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);
} else {
frameProcessorChain = null;
}
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
// has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set(
frameProcessorSizes.size() - 1,
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
frameProcessorChain =
FrameProcessorChain.create(
context,
inputFormat.pixelWidthHeightRatio,
frameProcessors,
frameProcessorSizes,
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);
decoder =
decoderFactory.createForVideoDecoding(
inputFormat,
frameProcessorChain == null
? encoder.getInputSurface()
: frameProcessorChain.createInputSurface());
inputFormat, frameProcessorChain.createInputSurface());
}
@Override
......@@ -154,15 +143,13 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public boolean processData() throws TransformationException {
if (frameProcessorChain != null) {
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
}
return false;
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
}
return false;
}
if (decoder.isEnded()) {
return false;
......@@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure;
canProcessMoreDataImmediately = processDataDefault();
}
if (decoder.isEnded()) {
if (frameProcessorChain != null) {
frameProcessorChain.signalEndOfInputStream();
} else {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
return false;
}
frameProcessorChain.signalEndOfInputStream();
}
return canProcessMoreDataImmediately;
}
......@@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure;
*/
private boolean processDataDefault() throws TransformationException {
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
if (frameProcessorChain.hasPendingFrames()) {
return false;
}
return maybeProcessDecoderOutput();
......@@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void release() {
if (frameProcessorChain != null) {
frameProcessorChain.release();
}
frameProcessorChain.release();
decoder.release();
encoder.release();
}
......@@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure;
.build();
}
/** Always use {@link FrameProcessorChain} to work around device-specific encoder issues. */
private static boolean shouldAlwaysUseFrameProcessorChain() {
switch (Util.MODEL) {
case "XT1635-02":
case "Nexus 5":
return true;
default:
return false;
}
}
/**
* Feeds at most one decoder output frame to the next step of the pipeline.
*
......@@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure;
return false;
}
if (frameProcessorChain != null) {
frameProcessorChain.registerInputFrame();
}
frameProcessorChain.registerInputFrame();
decoder.releaseOutputBuffer(/* render= */ true);
return true;
}
......
libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java
......@@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
......@@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
......@@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
}
......@@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}
......
libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java
......@@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
......@@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
......@@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
......@@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2);
}
......@@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
}
......@@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight);
assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment