Skip to content

SDK / exoplayer

Go to a project
Commit ecfbc65a by hschlueter Committed by Ian Baker
Options

Convert FrameEditor to a FrameProcessorChain. 

The FrameProcessorChain manages a List<GlFrameProcessor>.
FrameProcessorChainDataProcessingTest now tests chaining ScaleToFit-
and AdvancedFrameProcessors.

PiperOrigin-RevId: 436468037
parent 5b4abc31
Expand all
Inline Side-by-side
Showing with 173 additions and 59 deletions
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
......@@ -345,6 +345,7 @@ public final class GlUtil {
* @param height of the new texture in pixels
*/
public static int createTexture(int width, int height) {
assertValidTextureSize(width, height);
int texId = generateAndBindTexture(GLES20.GL_TEXTURE_2D);
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glTexImage2D(
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java
......@@ -44,7 +44,7 @@ import org.junit.runner.RunWith;
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
* as recommended in {@link FrameEditorDataProcessingTest}.
* as recommended in {@link FrameProcessorChainPixelTest}.
*/
@RunWith(AndroidJUnit4.class)
public final class AdvancedFrameProcessorPixelTest {
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java
......@@ -39,8 +39,8 @@ import java.io.InputStream;
import java.nio.ByteBuffer;
/**
* Utilities for instrumentation tests for the {@link FrameEditor} and {@link GlFrameProcessor
* GlFrameProcessors}.
* Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link
* GlFrameProcessor GlFrameProcessors}.
*/
public class BitmapTestUtil {
......@@ -53,6 +53,10 @@ public class BitmapTestUtil {
"media/bitmap/sample_mp4_first_frame_translate_right.png";
public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_scale_narrow.png";
public static final String ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate_then_translate.png";
public static final String TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_translate_then_rotate.png";
public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate90.png";
public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
......@@ -63,13 +67,15 @@ public class BitmapTestUtil {
* Maximum allowed average pixel difference between the expected and actual edited images for the
* test to pass. The value is chosen so that differences in decoder behavior across emulator
* versions don't affect whether the test passes for most emulators, but substantial distortions
* introduced by changes in the behavior of the frame editor will cause the test to fail.
* introduced by changes in the behavior of the {@link GlFrameProcessor GlFrameProcessors} will
* cause the test to fail.
*
* <p>To run this test on physical devices, please use a value of 5f, rather than 0.1f. This
* higher value will ignore some very small errors, but will allow for some differences caused by
* graphics implementations to be ignored. When the difference is close to the threshold, manually
* inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a
* difference in the codec or graphics implementation as opposed to a FrameEditor issue.
* difference in the codec or graphics implementation as opposed to a {@link GlFrameProcessor}
* issue.
*/
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java → libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
......@@ -20,19 +20,21 @@ import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.util.Size;
import android.view.Surface;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Test for {@link FrameEditor#create(Context, int, int, int, int, float, GlFrameProcessor, Surface,
* boolean, Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
* Test for {@link FrameProcessorChain#create(Context, float, List, List, Surface, boolean,
* Transformer.DebugViewProvider) creating} a {@link FrameProcessorChain}.
*/
@RunWith(AndroidJUnit4.class)
public final class FrameEditorTest {
public final class FrameProcessorChainTest {
// TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
// wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
......@@ -41,15 +43,12 @@ public final class FrameEditorTest {
throws TransformationException {
Context context = getApplicationContext();
FrameEditor.create(
FrameProcessorChain.create(
context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1,
new AdvancedFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
}
......@@ -62,15 +61,12 @@ public final class FrameEditorTest {
assertThrows(
TransformationException.class,
() ->
FrameEditor.create(
FrameProcessorChain.create(
context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 2,
new AdvancedFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE));
......
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
......@@ -32,6 +32,9 @@ import java.io.IOException;
* </ol>
*/
/* package */ interface GlFrameProcessor {
// TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
// using a placeholder surface until the encoder surface is known. If so, convert
// configureOutputSize to a simple getter.
/**
* Returns the output {@link Size} of frames processed through {@link
......
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
......@@ -27,9 +27,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be
......@@ -173,15 +171,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/
@RequiresNonNull("adjustedTransformationMatrix")
public boolean shouldProcess() {
checkStateNotNull(adjustedTransformationMatrix);
return inputWidth != outputWidth
|| inputHeight != outputHeight
|| !adjustedTransformationMatrix.isIdentity();
}
@Override
@EnsuresNonNull("adjustedTransformationMatrix")
public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
......@@ -191,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int displayHeight = inputHeight;
if (!transformationMatrix.isIdentity()) {
float inputAspectRatio = (float) inputWidth / inputHeight;
// Scale frames by inputAspectRatio, to account for FrameEditor's normalized device
// Scale frames by inputAspectRatio, to account for FrameProcessorChain's normalized device
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular
// display of input pixels during transformations (ex. rotations). With scaling,
// transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to
......
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
......@@ -274,15 +274,15 @@ public final class TransformationException extends Exception {
}
/**
* Creates an instance for a {@link FrameEditor} related exception.
* Creates an instance for a {@link FrameProcessorChain} related exception.
*
* @param cause The cause of the failure.
* @param errorCode See {@link #errorCode}.
* @return The created instance.
*/
/* package */ static TransformationException createForFrameEditor(
/* package */ static TransformationException createForFrameProcessorChain(
Throwable cause, int errorCode) {
return new TransformationException("FrameEditor error", cause, errorCode);
return new TransformationException("FrameProcessorChain error", cause, errorCode);
}
/**
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
......@@ -28,6 +28,8 @@ import androidx.annotation.RequiresApi;
import androidx.media3.common.Format;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure;
......@@ -40,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder;
@Nullable private final FrameEditor frameEditor;
@Nullable private final FrameProcessorChain frameProcessorChain;
private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer;
......@@ -74,14 +76,18 @@ import org.checkerframework.dataflow.qual.Pure;
.setRotationDegrees(transformationRequest.rotationDegrees)
.setResolution(transformationRequest.outputHeight)
.build();
Size requestedEncoderDimensions =
scaleToFitFrameProcessor.configureOutputSize(decodedWidth, decodedHeight);
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
ImmutableList<GlFrameProcessor> frameProcessors = ImmutableList.of(scaleToFitFrameProcessor);
List<Size> frameProcessorSizes =
FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors);
Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes);
// TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor.
outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees();
Format requestedEncoderFormat =
new Format.Builder()
.setWidth(requestedEncoderDimensions.getWidth())
.setHeight(requestedEncoderDimensions.getHeight())
.setWidth(requestedEncoderSize.getWidth())
.setHeight(requestedEncoderSize.getHeight())
.setRotationDegrees(0)
.setFrameRate(inputFormat.frameRate)
.setSampleMimeType(
......@@ -104,26 +110,30 @@ import org.checkerframework.dataflow.qual.Pure;
|| inputFormat.width != encoderSupportedFormat.width
|| scaleToFitFrameProcessor.shouldProcess()
|| shouldAlwaysUseFrameEditor()) {
frameEditor =
FrameEditor.create(
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
// has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set(
frameProcessorSizes.size() - 1,
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
frameProcessorChain =
FrameProcessorChain.create(
context,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
/* outputWidth= */ encoderSupportedFormat.width,
/* outputHeight= */ encoderSupportedFormat.height,
inputFormat.pixelWidthHeightRatio,
scaleToFitFrameProcessor,
frameProcessors,
frameProcessorSizes,
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);
} else {
frameEditor = null;
frameProcessorChain = null;
}
decoder =
decoderFactory.createForVideoDecoding(
inputFormat,
frameEditor == null ? encoder.getInputSurface() : frameEditor.createInputSurface());
frameProcessorChain == null
? encoder.getInputSurface()
: frameProcessorChain.createInputSurface());
}
@Override
......@@ -139,9 +149,9 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public boolean processData() throws TransformationException {
if (frameEditor != null) {
frameEditor.getAndRethrowBackgroundExceptions();
if (frameEditor.isEnded()) {
if (frameProcessorChain != null) {
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
......@@ -163,8 +173,8 @@ import org.checkerframework.dataflow.qual.Pure;
canProcessMoreDataImmediately = processDataDefault();
}
if (decoder.isEnded()) {
if (frameEditor != null) {
frameEditor.signalEndOfInputStream();
if (frameProcessorChain != null) {
frameProcessorChain.signalEndOfInputStream();
} else {
encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true;
......@@ -179,8 +189,8 @@ import org.checkerframework.dataflow.qual.Pure;
*
* <p>In this method the decoder could decode multiple frames in one invocation; as compared to
* {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently,
* if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up
* in the decoder's output surface.
* if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are
* queued up in the decoder's output surface.
*
* <p>Prior to API 29, decoders may drop frames to keep their output surface from growing out of
* bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
......@@ -195,12 +205,12 @@ import org.checkerframework.dataflow.qual.Pure;
/**
* Processes at most one input frame and returns whether a frame was processed.
*
* <p>Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link
* FrameEditor} has finished processing the previous frame.
* <p>Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link
* FrameProcessorChain} has finished processing the previous frame.
*/
private boolean processDataDefault() throws TransformationException {
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
if (frameEditor != null && frameEditor.hasPendingFrames()) {
if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
return false;
}
return maybeProcessDecoderOutput();
......@@ -240,8 +250,8 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void release() {
if (frameEditor != null) {
frameEditor.release();
if (frameProcessorChain != null) {
frameProcessorChain.release();
}
decoder.release();
encoder.release();
......@@ -299,8 +309,8 @@ import org.checkerframework.dataflow.qual.Pure;
return false;
}
if (frameEditor != null) {
frameEditor.registerInputFrame();
if (frameProcessorChain != null) {
frameProcessorChain.registerInputFrame();
}
decoder.releaseOutputBuffer(/* render= */ true);
return true;
......
libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java 0 → 100644
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static com.google.common.truth.Truth.assertThat;
import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Robolectric tests for {@link FrameProcessorChain}.
*
* <p>See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the
* androidTest directory for instrumentation tests.
*/
@RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest {
@Test
public void configureOutputDimensions_withEmptyList_returnsInputSize() {
Size inputSize = new Size(200, 100);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of());
assertThat(sizes).containsExactly(inputSize);
}
@Test
public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() {
Size inputSize = new Size(200, 100);
Size outputSize = new Size(300, 250);
GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor));
assertThat(sizes).containsExactly(inputSize, outputSize).inOrder();
}
@Test
public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() {
Size inputSize = new Size(200, 100);
Size outputSize1 = new Size(300, 250);
Size outputSize2 = new Size(400, 244);
Size outputSize3 = new Size(150, 160);
GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1);
GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2);
GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(),
inputSize.getHeight(),
ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3));
assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder();
}
private static class FakeFrameProcessor implements GlFrameProcessor {
private final Size outputSize;
private FakeFrameProcessor(Size outputSize) {
this.outputSize = outputSize;
}
@Override
public Size configureOutputSize(int inputWidth, int inputHeight) {
return outputSize;
}
@Override
public void initialize(int inputTexId) {}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {}
@Override
public void release() {}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment