Skip to content

SDK / exoplayer

Go to a project
Commit 2a363ac3 by huangdarwin Committed by Ian Baker
Options

Transformer: Always use FrameProcessorChain when decoding. 

This allows us to bypass many device-specific issues, that only occur when
decoding directly to an encoder surface, without OpenGL. This also allows us
to maintain fewer code branches, which require additional testing to verify
correctness.

PiperOrigin-RevId: 437003138
parent e8b0971f
Inline Side-by-side
Showing with 8 additions and 113 deletions
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
...@@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; ...@@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import android.content.Context; import android.content.Context;
import androidx.media3.common.MimeTypes;
import androidx.test.core.app.ApplicationProvider; import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test; import org.junit.Test;
...@@ -32,38 +31,9 @@ import org.junit.runner.RunWith; ...@@ -32,38 +31,9 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public class TransformerEndToEndTest { public class TransformerEndToEndTest {
private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm";
private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4"; private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4";
@Test @Test
public void videoTranscoding_completesWithConsistentFrameCount() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
FrameCountingMuxer.Factory muxerFactory =
new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory());
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build())
.setMuxerFactory(muxerFactory)
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
// Result of the following command:
// ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm
int expectedFrameCount = 82;
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(
/* testId= */ "videoTranscoding_completesWithConsistentFrameCount",
VP9_VIDEO_URI_STRING);
FrameCountingMuxer frameCountingMuxer =
checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated());
assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount);
}
@Test
public void videoEditing_completesWithConsistentFrameCount() throws Exception { public void videoEditing_completesWithConsistentFrameCount() throws Exception {
Context context = ApplicationProvider.getApplicationContext(); Context context = ApplicationProvider.getApplicationContext();
FrameCountingMuxer.Factory muxerFactory = FrameCountingMuxer.Factory muxerFactory =
......
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
...@@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
private int outputHeight;
private int outputRotationDegrees; private int outputRotationDegrees;
private @MonotonicNonNull Matrix transformationMatrix; private @MonotonicNonNull Matrix transformationMatrix;
...@@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputWidth = C.LENGTH_UNSET; inputWidth = C.LENGTH_UNSET;
inputHeight = C.LENGTH_UNSET; inputHeight = C.LENGTH_UNSET;
outputHeight = C.LENGTH_UNSET;
outputRotationDegrees = C.LENGTH_UNSET; outputRotationDegrees = C.LENGTH_UNSET;
} }
...@@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return outputRotationDegrees; return outputRotationDegrees;
} }
/**
* Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame.
*
* <p>The {@code PresentationFrameProcessor} should only be used if this returns true.
*
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/
public boolean shouldProcess() {
checkStateNotNull(transformationMatrix);
return inputHeight != outputHeight || !transformationMatrix.isIdentity();
}
@Override @Override
public Size configureOutputSize(int inputWidth, int inputHeight) { public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth; this.inputWidth = inputWidth;
...@@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
displayHeight = requestedHeight; displayHeight = requestedHeight;
} }
int outputWidth;
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// frame before encoding, so the encoded frame's width >= height, and set // frame before encoding, so the encoded frame's width >= height, and set
// outputRotationDegrees to ensure the frame is displayed in the correct orientation. // outputRotationDegrees to ensure the frame is displayed in the correct orientation.
if (displayHeight > displayWidth) { if (displayHeight > displayWidth) {
outputRotationDegrees = 90; outputRotationDegrees = 90;
outputWidth = displayHeight; // TODO(b/201293185): After fragment shader transformations are implemented, put
outputHeight = displayWidth; // postRotate in a later GlFrameProcessor.
// TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in
// a later GlFrameProcessor.
transformationMatrix.postRotate(outputRotationDegrees); transformationMatrix.postRotate(outputRotationDegrees);
return new Size(displayHeight, displayWidth);
} else { } else {
outputRotationDegrees = 0; outputRotationDegrees = 0;
outputWidth = displayWidth; return new Size(displayWidth, displayHeight);
outputHeight = displayHeight;
} }
return new Size(outputWidth, outputHeight);
} }
@Override @Override
......
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
...@@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputHeight = C.LENGTH_UNSET; inputHeight = C.LENGTH_UNSET;
} }
/**
* Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame.
*
* <p>The ScaleToFitFrameProcessor should only be used if this returns true.
*
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/
public boolean shouldProcess() {
checkStateNotNull(adjustedTransformationMatrix);
return !transformationMatrix.isIdentity();
}
@Override @Override
public Size configureOutputSize(int inputWidth, int inputHeight) { public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth; this.inputWidth = inputWidth;
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
...@@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final DecoderInputBuffer decoderInputBuffer; private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder; private final Codec decoder;
@Nullable private final FrameProcessorChain frameProcessorChain; private final FrameProcessorChain frameProcessorChain;
private final Codec encoder; private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer; private final DecoderInputBuffer encoderOutputBuffer;
...@@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight = int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor = ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder(context) new ScaleToFitFrameProcessor.Builder(context)
.setScale(transformationRequest.scaleX, transformationRequest.scaleY) .setScale(transformationRequest.scaleX, transformationRequest.scaleY)
...@@ -109,12 +110,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -109,12 +110,6 @@ import org.checkerframework.dataflow.qual.Pure;
requestedEncoderFormat, requestedEncoderFormat,
encoderSupportedFormat)); encoderSupportedFormat));
if (transformationRequest.enableHdrEditing
|| inputFormat.height != encoderSupportedFormat.height
|| inputFormat.width != encoderSupportedFormat.width
|| scaleToFitFrameProcessor.shouldProcess()
|| presentationFrameProcessor.shouldProcess()
|| shouldAlwaysUseFrameProcessorChain()) {
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
// has to change due to encoder fallback or append another GlFrameProcessor. // has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set( frameProcessorSizes.set(
...@@ -129,16 +124,10 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -129,16 +124,10 @@ import org.checkerframework.dataflow.qual.Pure;
/* outputSurface= */ encoder.getInputSurface(), /* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing, transformationRequest.enableHdrEditing,
debugViewProvider); debugViewProvider);
} else {
frameProcessorChain = null;
}
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(
inputFormat, inputFormat, frameProcessorChain.createInputSurface());
frameProcessorChain == null
? encoder.getInputSurface()
: frameProcessorChain.createInputSurface());
} }
@Override @Override
...@@ -154,7 +143,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -154,7 +143,6 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() throws TransformationException {
if (frameProcessorChain != null) {
frameProcessorChain.getAndRethrowBackgroundExceptions(); frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) { if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) { if (!signaledEndOfStreamToEncoder) {
...@@ -163,7 +151,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -163,7 +151,6 @@ import org.checkerframework.dataflow.qual.Pure;
} }
return false; return false;
} }
}
if (decoder.isEnded()) { if (decoder.isEnded()) {
return false; return false;
} }
...@@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure;
canProcessMoreDataImmediately = processDataDefault(); canProcessMoreDataImmediately = processDataDefault();
} }
if (decoder.isEnded()) { if (decoder.isEnded()) {
if (frameProcessorChain != null) {
frameProcessorChain.signalEndOfInputStream(); frameProcessorChain.signalEndOfInputStream();
} else {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
return false;
}
} }
return canProcessMoreDataImmediately; return canProcessMoreDataImmediately;
} }
...@@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure;
*/ */
private boolean processDataDefault() throws TransformationException { private boolean processDataDefault() throws TransformationException {
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) { if (frameProcessorChain.hasPendingFrames()) {
return false; return false;
} }
return maybeProcessDecoderOutput(); return maybeProcessDecoderOutput();
...@@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public void release() { public void release() {
if (frameProcessorChain != null) {
frameProcessorChain.release(); frameProcessorChain.release();
}
decoder.release(); decoder.release();
encoder.release(); encoder.release();
} }
...@@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure;
.build(); .build();
} }
/** Always use {@link FrameProcessorChain} to work around device-specific encoder issues. */
private static boolean shouldAlwaysUseFrameProcessorChain() {
switch (Util.MODEL) {
case "XT1635-02":
case "Nexus 5":
return true;
default:
return false;
}
}
/** /**
* Feeds at most one decoder output frame to the next step of the pipeline. * Feeds at most one decoder output frame to the next step of the pipeline.
* *
...@@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure;
return false; return false;
} }
if (frameProcessorChain != null) {
frameProcessorChain.registerInputFrame(); frameProcessorChain.registerInputFrame();
}
decoder.releaseOutputBuffer(/* render= */ true); decoder.releaseOutputBuffer(/* render= */ true);
return true; return true;
} }
......
libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java
...@@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest { ...@@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
} }
...@@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest { ...@@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
} }
...@@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest { ...@@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
} }
...@@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest { ...@@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest {
Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
} }
......
libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java
...@@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
} }
...@@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f)); assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
} }
...@@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2); assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
} }
...@@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2); assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2);
} }
...@@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
} }
...@@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest { ...@@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest {
Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight); assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight);
assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight); assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment