Skip to content

SDK / exoplayer

Go to a project
Commit ecfbc65a by hschlueter Committed by Ian Baker
Options

Convert FrameEditor to a FrameProcessorChain. 

The FrameProcessorChain manages a List<GlFrameProcessor>.
FrameProcessorChainDataProcessingTest now tests chaining ScaleToFit-
and AdvancedFrameProcessors.

PiperOrigin-RevId: 436468037
parent 5b4abc31
Inline Side-by-side
Showing with 405 additions and 225 deletions
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
...@@ -345,6 +345,7 @@ public final class GlUtil { ...@@ -345,6 +345,7 @@ public final class GlUtil {
* @param height of the new texture in pixels * @param height of the new texture in pixels
*/ */
public static int createTexture(int width, int height) { public static int createTexture(int width, int height) {
assertValidTextureSize(width, height);
int texId = generateAndBindTexture(GLES20.GL_TEXTURE_2D); int texId = generateAndBindTexture(GLES20.GL_TEXTURE_2D);
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4); ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glTexImage2D( GLES20.glTexImage2D(
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java
...@@ -44,7 +44,7 @@ import org.junit.runner.RunWith; ...@@ -44,7 +44,7 @@ import org.junit.runner.RunWith;
* <p>Expected images are taken from an emulator, so tests on different emulators or physical * <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link * devices may fail. To test on other devices, please increase the {@link
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
* as recommended in {@link FrameEditorDataProcessingTest}. * as recommended in {@link FrameProcessorChainPixelTest}.
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class AdvancedFrameProcessorPixelTest { public final class AdvancedFrameProcessorPixelTest {
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java
...@@ -39,8 +39,8 @@ import java.io.InputStream; ...@@ -39,8 +39,8 @@ import java.io.InputStream;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/** /**
* Utilities for instrumentation tests for the {@link FrameEditor} and {@link GlFrameProcessor * Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link
* GlFrameProcessors}. * GlFrameProcessor GlFrameProcessors}.
*/ */
public class BitmapTestUtil { public class BitmapTestUtil {
...@@ -53,6 +53,10 @@ public class BitmapTestUtil { ...@@ -53,6 +53,10 @@ public class BitmapTestUtil {
"media/bitmap/sample_mp4_first_frame_translate_right.png"; "media/bitmap/sample_mp4_first_frame_translate_right.png";
public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING = public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_scale_narrow.png"; "media/bitmap/sample_mp4_first_frame_scale_narrow.png";
public static final String ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate_then_translate.png";
public static final String TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_translate_then_rotate.png";
public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING = public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate90.png"; "media/bitmap/sample_mp4_first_frame_rotate90.png";
public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING = public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
...@@ -63,13 +67,15 @@ public class BitmapTestUtil { ...@@ -63,13 +67,15 @@ public class BitmapTestUtil {
* Maximum allowed average pixel difference between the expected and actual edited images for the * Maximum allowed average pixel difference between the expected and actual edited images for the
* test to pass. The value is chosen so that differences in decoder behavior across emulator * test to pass. The value is chosen so that differences in decoder behavior across emulator
* versions don't affect whether the test passes for most emulators, but substantial distortions * versions don't affect whether the test passes for most emulators, but substantial distortions
* introduced by changes in the behavior of the frame editor will cause the test to fail. * introduced by changes in the behavior of the {@link GlFrameProcessor GlFrameProcessors} will
* cause the test to fail.
* *
* <p>To run this test on physical devices, please use a value of 5f, rather than 0.1f. This * <p>To run this test on physical devices, please use a value of 5f, rather than 0.1f. This
* higher value will ignore some very small errors, but will allow for some differences caused by * higher value will ignore some very small errors, but will allow for some differences caused by
* graphics implementations to be ignored. When the difference is close to the threshold, manually * graphics implementations to be ignored. When the difference is close to the threshold, manually
* inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a * inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a
* difference in the codec or graphics implementation as opposed to a FrameEditor issue. * difference in the codec or graphics implementation as opposed to a {@link GlFrameProcessor}
* issue.
*/ */
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f; public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java → libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
...@@ -20,11 +20,12 @@ import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_S ...@@ -20,11 +20,12 @@ import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_S
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static java.util.Arrays.asList;
import android.content.Context; import android.content.Context;
import android.content.res.AssetFileDescriptor; import android.content.res.AssetFileDescriptor;
...@@ -40,14 +41,16 @@ import android.util.Size; ...@@ -40,14 +41,16 @@ import android.util.Size;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.media3.common.MimeTypes; import androidx.media3.common.MimeTypes;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.Iterables;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
/** /**
* Pixel test for frame processing via {@link FrameEditor}. * Pixel test for frame processing via {@link FrameProcessorChain}.
* *
* <p>Expected images are taken from an emulator, so tests on different emulators or physical * <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link * devices may fail. To test on other devices, please increase the {@link
...@@ -55,41 +58,35 @@ import org.junit.runner.RunWith; ...@@ -55,41 +58,35 @@ import org.junit.runner.RunWith;
* bitmaps. * bitmaps.
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameEditorDataProcessingTest { public final class FrameProcessorChainPixelTest {
// TODO(b/214975934): Once FrameEditor is converted to a FrameProcessorChain, replace these tests
// with a test for a few example combinations of GlFrameProcessors rather than testing all use
// cases of TransformationFrameProcessor.
/** Input video of which we only use the first frame. */ /** Input video of which we only use the first frame. */
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
/** Timeout for dequeueing buffers from the codec, in microseconds. */ /** Timeout for dequeueing buffers from the codec, in microseconds. */
private static final int DEQUEUE_TIMEOUT_US = 5_000_000; private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
/** /**
* Time to wait for the decoded frame to populate the frame editor's input surface and the frame * Time to wait for the decoded frame to populate the {@link FrameProcessorChain}'s input surface
* editor to finish processing the frame, in milliseconds. * and the {@link FrameProcessorChain} to finish processing the frame, in milliseconds.
*/ */
private static final int FRAME_PROCESSING_WAIT_MS = 1000; private static final int FRAME_PROCESSING_WAIT_MS = 2000;
/** The ratio of width over height, for each pixel in a frame. */ /** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
private @MonotonicNonNull FrameEditor frameEditor; private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
private @MonotonicNonNull ImageReader frameEditorOutputImageReader; private @MonotonicNonNull ImageReader outputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat; private @MonotonicNonNull MediaFormat mediaFormat;
@After @After
public void release() { public void release() {
if (frameEditor != null) { if (frameProcessorChain != null) {
frameEditor.release(); frameProcessorChain.release();
} }
} }
@Test @Test
public void processData_noEdits_producesExpectedOutput() throws Exception { public void processData_noEdits_producesExpectedOutput() throws Exception {
final String testId = "processData_noEdits"; final String testId = "processData_noEdits";
Matrix identityMatrix = new Matrix(); setUpAndPrepareFirstFrame();
GlFrameProcessor glFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd(); Bitmap actualBitmap = processFirstFrameAndEnd();
...@@ -104,8 +101,9 @@ public final class FrameEditorDataProcessingTest { ...@@ -104,8 +101,9 @@ public final class FrameEditorDataProcessingTest {
} }
@Test @Test
public void processData_translateRight_producesExpectedOutput() throws Exception { public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput()
final String testId = "processData_translateRight"; throws Exception {
final String testId = "processData_withAdvancedFrameProcessor_translateRight";
Matrix translateRightMatrix = new Matrix(); Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor glFrameProcessor = GlFrameProcessor glFrameProcessor =
...@@ -126,15 +124,20 @@ public final class FrameEditorDataProcessingTest { ...@@ -126,15 +124,20 @@ public final class FrameEditorDataProcessingTest {
} }
@Test @Test
public void processData_scaleNarrow_producesExpectedOutput() throws Exception { public void processData_withAdvancedAndScaleToFitFrameProcessors_producesExpectedOutput()
final String testId = "processData_scaleNarrow"; throws Exception {
Matrix scaleNarrowMatrix = new Matrix(); final String testId = "processData_withAdvancedAndScaleToFitFrameProcessors";
scaleNarrowMatrix.postScale(.5f, 1.2f); Matrix translateRightMatrix = new Matrix();
GlFrameProcessor glFrameProcessor = translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix); GlFrameProcessor translateRightFrameProcessor =
setUpAndPrepareFirstFrame(glFrameProcessor); new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
GlFrameProcessor rotate45FrameProcessor =
new ScaleToFitFrameProcessor.Builder(getApplicationContext())
.setRotationDegrees(45)
.build();
setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor);
Bitmap expectedBitmap = Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd(); Bitmap actualBitmap = processFirstFrameAndEnd();
...@@ -148,14 +151,20 @@ public final class FrameEditorDataProcessingTest { ...@@ -148,14 +151,20 @@ public final class FrameEditorDataProcessingTest {
} }
@Test @Test
public void processData_rotate90_producesExpectedOutput() throws Exception { public void processData_withScaleToFitAndAdvancedFrameProcessors_producesExpectedOutput()
final String testId = "processData_rotate90"; throws Exception {
Matrix rotate90Matrix = new Matrix(); final String testId = "processData_withScaleToFitAndAdvancedFrameProcessors";
rotate90Matrix.postRotate(/* degrees= */ 90); GlFrameProcessor rotate45FrameProcessor =
GlFrameProcessor glFrameProcessor = new ScaleToFitFrameProcessor.Builder(getApplicationContext())
new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix); .setRotationDegrees(45)
setUpAndPrepareFirstFrame(glFrameProcessor); .build();
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor translateRightFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd(); Bitmap actualBitmap = processFirstFrameAndEnd();
...@@ -169,8 +178,9 @@ public final class FrameEditorDataProcessingTest { ...@@ -169,8 +178,9 @@ public final class FrameEditorDataProcessingTest {
} }
@Test @Test
public void processData_requestOutputHeight_producesExpectedOutput() throws Exception { public void processData_withScaleToFitFrameProcessor_requestOutputHeight_producesExpectedOutput()
final String testId = "processData_requestOutputHeight"; throws Exception {
final String testId = "processData_withScaleToFitFrameProcessor_requestOutputHeight";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image. // test that rotation doesn't distort the image.
...@@ -192,8 +202,9 @@ public final class FrameEditorDataProcessingTest { ...@@ -192,8 +202,9 @@ public final class FrameEditorDataProcessingTest {
} }
@Test @Test
public void processData_rotate45_scaleToFit_producesExpectedOutput() throws Exception { public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
final String testId = "processData_rotate45_scaleToFit"; throws Exception {
final String testId = "processData_withScaleToFitFrameProcessor_rotate45";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image. // test that rotation doesn't distort the image.
...@@ -218,12 +229,13 @@ public final class FrameEditorDataProcessingTest { ...@@ -218,12 +229,13 @@ public final class FrameEditorDataProcessingTest {
/** /**
* Set up and prepare the first frame from an input video, as well as relevant test * Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link FrameEditor}, and may be accessed on * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
* the {@link FrameEditor}'s output {@code frameEditorOutputImageReader}. * accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
* *
* @param glFrameProcessor The frame processor that will apply changes to the input frame. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
* to the input frame.
*/ */
private void setUpAndPrepareFirstFrame(GlFrameProcessor glFrameProcessor) throws Exception { private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception {
// Set up the extractor to read the first video frame and get its format. // Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor(); MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null; @Nullable MediaCodec mediaCodec = null;
...@@ -240,31 +252,35 @@ public final class FrameEditorDataProcessingTest { ...@@ -240,31 +252,35 @@ public final class FrameEditorDataProcessingTest {
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
Size outputSize = glFrameProcessor.configureOutputSize(inputWidth, inputHeight); List<GlFrameProcessor> frameProcessorsList = asList(frameProcessors);
int outputWidth = outputSize.getWidth(); List<Size> sizes =
int outputHeight = outputSize.getHeight(); FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList);
frameEditorOutputImageReader = assertThat(sizes).isNotEmpty();
outputImageReader =
ImageReader.newInstance( ImageReader.newInstance(
outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1); Iterables.getLast(sizes).getWidth(),
frameEditor = Iterables.getLast(sizes).getHeight(),
FrameEditor.create( PixelFormat.RGBA_8888,
/* maxImages= */ 1);
frameProcessorChain =
FrameProcessorChain.create(
context, context,
inputWidth,
inputHeight,
outputWidth,
outputHeight,
PIXEL_WIDTH_HEIGHT_RATIO, PIXEL_WIDTH_HEIGHT_RATIO,
glFrameProcessor, frameProcessorsList,
frameEditorOutputImageReader.getSurface(), sizes,
outputImageReader.getSurface(),
/* enableExperimentalHdrEditing= */ false, /* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE); Transformer.DebugViewProvider.NONE);
frameEditor.registerInputFrame(); frameProcessorChain.registerInputFrame();
// Queue the first video frame from the extractor. // Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure( mediaCodec.configure(
mediaFormat, frameEditor.createInputSurface(), /* crypto= */ null, /* flags= */ 0); mediaFormat,
frameProcessorChain.createInputSurface(),
/* crypto= */ null,
/* flags= */ 0);
mediaCodec.start(); mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
...@@ -305,14 +321,15 @@ public final class FrameEditorDataProcessingTest { ...@@ -305,14 +321,15 @@ public final class FrameEditorDataProcessingTest {
} }
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException { private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException {
checkNotNull(frameEditor).signalEndOfInputStream(); checkNotNull(frameProcessorChain).signalEndOfInputStream();
Thread.sleep(FRAME_PROCESSING_WAIT_MS); Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameEditor.isEnded()).isTrue(); assertThat(frameProcessorChain.isEnded()).isTrue();
frameEditor.getAndRethrowBackgroundExceptions(); frameProcessorChain.getAndRethrowBackgroundExceptions();
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage); Bitmap actualBitmap =
editorOutputImage.close(); BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(frameProcessorChainOutputImage);
frameProcessorChainOutputImage.close();
return actualBitmap; return actualBitmap;
} }
} }
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java → libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
...@@ -20,19 +20,21 @@ import static com.google.common.truth.Truth.assertThat; ...@@ -20,19 +20,21 @@ import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertThrows;
import android.content.Context; import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.util.Size;
import android.view.Surface; import android.view.Surface;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
/** /**
* Test for {@link FrameEditor#create(Context, int, int, int, int, float, GlFrameProcessor, Surface, * Test for {@link FrameProcessorChain#create(Context, float, List, List, Surface, boolean,
* boolean, Transformer.DebugViewProvider) creating} a {@link FrameEditor}. * Transformer.DebugViewProvider) creating} a {@link FrameProcessorChain}.
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameEditorTest { public final class FrameProcessorChainTest {
// TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a // TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
// wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20. // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
...@@ -41,15 +43,12 @@ public final class FrameEditorTest { ...@@ -41,15 +43,12 @@ public final class FrameEditorTest {
throws TransformationException { throws TransformationException {
Context context = getApplicationContext(); Context context = getApplicationContext();
FrameEditor.create( FrameProcessorChain.create(
context, context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1, /* pixelWidthHeightRatio= */ 1,
new AdvancedFrameProcessor(context, new Matrix()), /* frameProcessors= */ ImmutableList.of(),
new Surface(new SurfaceTexture(false)), /* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false, /* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE); Transformer.DebugViewProvider.NONE);
} }
...@@ -62,15 +61,12 @@ public final class FrameEditorTest { ...@@ -62,15 +61,12 @@ public final class FrameEditorTest {
assertThrows( assertThrows(
TransformationException.class, TransformationException.class,
() -> () ->
FrameEditor.create( FrameProcessorChain.create(
context, context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 2, /* pixelWidthHeightRatio= */ 2,
new AdvancedFrameProcessor(context, new Matrix()), /* frameProcessors= */ ImmutableList.of(),
new Surface(new SurfaceTexture(false)), /* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false, /* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE)); Transformer.DebugViewProvider.NONE));
......
libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java → libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
...@@ -15,8 +15,10 @@ ...@@ -15,8 +15,10 @@
*/ */
package androidx.media3.transformer; package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkState;
import static com.google.common.collect.Iterables.getLast;
import android.content.Context; import android.content.Context;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
...@@ -26,6 +28,7 @@ import android.opengl.EGLDisplay; ...@@ -26,6 +28,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.util.Size;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceView; import android.view.SurfaceView;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
...@@ -33,6 +36,8 @@ import androidx.media3.common.C; ...@@ -33,6 +36,8 @@ import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
...@@ -43,69 +48,90 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -43,69 +48,90 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* {@code FrameEditor} applies changes to individual video frames. * {@code FrameProcessorChain} applies changes to individual video frames.
* *
* <p>Input becomes available on its {@link #createInputSurface() input surface} asynchronously and * <p>Input becomes available on its {@link #createInputSurface() input surface} asynchronously and
* is processed on a background thread as it becomes available. All input frames should be {@link * is processed on a background thread as it becomes available. All input frames should be {@link
* #registerInputFrame() registered} before they are rendered to the input surface. {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully * #hasPendingFrames()} can be used to check whether there are frames that have not been fully
* processed yet. Output is written to its {@link #create(Context, int, int, int, int, float, * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link
* GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}. * #create(Context, float, List, List, Surface, boolean, Transformer.DebugViewProvider)}.
*/ */
/* package */ final class FrameEditor { /* package */ final class FrameProcessorChain {
static { static {
GlUtil.glAssertionsEnabled = true; GlUtil.glAssertionsEnabled = true;
} }
/** /**
* Returns a new {@code FrameEditor} for applying changes to individual frames. * Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor
* GlFrameProcessors}.
*
* @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}.
* @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}.
* @return A mutable {@link List} containing the input {@link Size} as well as the output {@link
* Size} of each {@link GlFrameProcessor}.
*/
// TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs
// to modify this list for encoder fallback.
public static List<Size> configureSizes(
int inputWidth, int inputHeight, List<GlFrameProcessor> frameProcessors) {
List<Size> sizes = new ArrayList<>(frameProcessors.size() + 1);
sizes.add(new Size(inputWidth, inputHeight));
for (int i = 0; i < frameProcessors.size(); i++) {
sizes.add(
frameProcessors
.get(i)
.configureOutputSize(getLast(sizes).getWidth(), getLast(sizes).getHeight()));
}
return sizes;
}
/**
* Returns a new {@code FrameProcessorChain} for applying changes to individual frames.
* *
* @param context A {@link Context}. * @param context A {@link Context}.
* @param inputWidth The input width in pixels.
* @param inputHeight The input height in pixels.
* @param outputWidth The output width in pixels.
* @param outputHeight The output height in pixels.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel. * @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)}
* configured}.
* @param sizes The input {@link Size} as well as the output {@link Size} of each {@link
* GlFrameProcessor}.
* @param outputSurface The {@link Surface}. * @param outputSurface The {@link Surface}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param debugViewProvider Provider for optional debug views to show intermediate output. * @param debugViewProvider Provider for optional debug views to show intermediate output.
* @return A configured {@code FrameEditor}. * @return A configured {@code FrameProcessorChain}.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL * files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components. * components.
*/ */
// TODO(b/214975934): Take a List<GlFrameProcessor> as input and rename FrameEditor to public static FrameProcessorChain create(
// FrameProcessorChain.
// TODO(b/218488308): Remove the need to input outputWidth and outputHeight into FrameEditor, that
// stems from encoder fallback resolution. This could maybe be input into the last
// GlFrameProcessor in the FrameEditor instead of being input directly into the FrameEditor.
public static FrameEditor create(
Context context, Context context,
int inputWidth,
int inputHeight,
int outputWidth,
int outputHeight,
float pixelWidthHeightRatio, float pixelWidthHeightRatio,
GlFrameProcessor transformationFrameProcessor, List<GlFrameProcessor> frameProcessors,
List<Size> sizes,
Surface outputSurface, Surface outputSurface,
boolean enableExperimentalHdrEditing, boolean enableExperimentalHdrEditing,
Transformer.DebugViewProvider debugViewProvider) Transformer.DebugViewProvider debugViewProvider)
throws TransformationException { throws TransformationException {
checkArgument(frameProcessors.size() + 1 == sizes.size());
if (pixelWidthHeightRatio != 1.0f) { if (pixelWidthHeightRatio != 1.0f) {
// TODO(b/211782176): Consider implementing support for non-square pixels. // TODO(b/211782176): Consider implementing support for non-square pixels.
throw TransformationException.createForFrameEditor( throw TransformationException.createForFrameProcessorChain(
new UnsupportedOperationException( new UnsupportedOperationException(
"Transformer's frame editor currently does not support frame edits on non-square" "Transformer's FrameProcessorChain currently does not support frame edits on"
+ " pixels. The pixelWidthHeightRatio is: " + " non-square pixels. The pixelWidthHeightRatio is: "
+ pixelWidthHeightRatio), + pixelWidthHeightRatio),
TransformationException.ERROR_CODE_GL_INIT_FAILED); TransformationException.ERROR_CODE_GL_INIT_FAILED);
} }
@Nullable @Nullable
SurfaceView debugSurfaceView = SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); debugViewProvider.getDebugPreviewSurfaceView(
getLast(sizes).getWidth(), getLast(sizes).getHeight());
int debugPreviewWidth; int debugPreviewWidth;
int debugPreviewHeight; int debugPreviewHeight;
if (debugSurfaceView != null) { if (debugSurfaceView != null) {
...@@ -120,51 +146,45 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -120,51 +146,45 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
Future<FrameEditor> frameEditorFuture = Future<FrameProcessorChain> frameProcessorChainFuture =
singleThreadExecutorService.submit( singleThreadExecutorService.submit(
() -> () ->
createOpenGlObjectsAndFrameEditor( createOpenGlObjectsAndFrameProcessorChain(
singleThreadExecutorService, singleThreadExecutorService,
externalCopyFrameProcessor, externalCopyFrameProcessor,
transformationFrameProcessor, frameProcessors,
sizes,
outputSurface, outputSurface,
inputWidth,
inputHeight,
outputWidth,
outputHeight,
enableExperimentalHdrEditing, enableExperimentalHdrEditing,
debugSurfaceView, debugSurfaceView,
debugPreviewWidth, debugPreviewWidth,
debugPreviewHeight)); debugPreviewHeight));
try { try {
return frameEditorFuture.get(); return frameProcessorChainFuture.get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
throw TransformationException.createForFrameEditor( throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED); e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw TransformationException.createForFrameEditor( throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED); e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} }
} }
/** /**
* Creates a {@code FrameEditor} and its OpenGL objects. * Creates a {@code FrameProcessorChain} and its OpenGL objects.
* *
* <p>As the {@code FrameEditor} will call OpenGL commands on the {@code * <p>As the {@code FrameProcessorChain} will call OpenGL commands on the {@code
* singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created * singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created
* on that thread. So this method should only be called on the {@code * on that thread. So this method should only be called on the {@code
* singleThreadExecutorService}'s thread. * singleThreadExecutorService}'s thread.
*/ */
private static FrameEditor createOpenGlObjectsAndFrameEditor( private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
ExecutorService singleThreadExecutorService, ExecutorService singleThreadExecutorService,
ExternalCopyFrameProcessor externalCopyFrameProcessor, ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor, List<GlFrameProcessor> frameProcessors,
List<Size> sizes,
Surface outputSurface, Surface outputSurface,
int inputWidth,
int inputHeight,
int outputWidth,
int outputHeight,
boolean enableExperimentalHdrEditing, boolean enableExperimentalHdrEditing,
@Nullable SurfaceView debugSurfaceView, @Nullable SurfaceView debugSurfaceView,
int debugPreviewWidth, int debugPreviewWidth,
...@@ -192,35 +212,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -192,35 +212,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
} }
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); GlUtil.focusEglSurface(
GlUtil.assertValidTextureSize(outputWidth, outputHeight); eglDisplay, eglContext, eglSurface, getLast(sizes).getWidth(), getLast(sizes).getHeight());
int inputExternalTexId = GlUtil.createExternalTexture(); int inputExternalTexId = GlUtil.createExternalTexture();
// TODO(b/214975934): Propagate output sizes through the chain of frame processors. externalCopyFrameProcessor.configureOutputSize(
externalCopyFrameProcessor.configureOutputSize(inputWidth, inputHeight); /* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight());
externalCopyFrameProcessor.initialize(/* inputTexId= */ inputExternalTexId); externalCopyFrameProcessor.initialize(inputExternalTexId);
int intermediateTexId = GlUtil.createTexture(inputWidth, inputHeight);
int frameBuffer = GlUtil.createFboForTexture(intermediateTexId); int[] framebuffers = new int[frameProcessors.size()];
transformationFrameProcessor.initialize(/* inputTexId= */ intermediateTexId); for (int i = 0; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight());
return new FrameEditor( framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
frameProcessors.get(i).initialize(inputTexId);
}
return new FrameProcessorChain(
singleThreadExecutorService, singleThreadExecutorService,
eglDisplay, eglDisplay,
eglContext, eglContext,
eglSurface, eglSurface,
externalCopyFrameProcessor, externalCopyFrameProcessor,
transformationFrameProcessor, frameProcessors,
inputWidth,
inputHeight,
inputExternalTexId, inputExternalTexId,
frameBuffer, framebuffers,
outputWidth, sizes,
outputHeight,
debugPreviewEglSurface, debugPreviewEglSurface,
debugPreviewWidth, debugPreviewWidth,
debugPreviewHeight); debugPreviewHeight);
} }
private static final String THREAD_NAME = "Transformer:FrameEditor"; private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
/** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */
private final ExecutorService singleThreadExecutorService; private final ExecutorService singleThreadExecutorService;
...@@ -235,25 +257,28 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -235,25 +257,28 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final EGLContext eglContext; private final EGLContext eglContext;
private final EGLSurface eglSurface; private final EGLSurface eglSurface;
private final ExternalCopyFrameProcessor externalCopyFrameProcessor; private final ExternalCopyFrameProcessor externalCopyFrameProcessor;
private final GlFrameProcessor transformationFrameProcessor; private final List<GlFrameProcessor> frameProcessors;
/**
/** Identifier of the external texture the {@code FrameEditor} reads its input from. */ * Identifiers of a framebuffer object associated with the intermediate textures that receive
* output from the previous {@link GlFrameProcessor}, and provide input for the following {@link
* GlFrameProcessor}.
*
* <p>The {@link ExternalCopyFrameProcessor} writes to the first framebuffer.
*/
private final int[] framebuffers;
/**
* The input {@link Size}, i.e., the output {@link Size} of the {@link
* ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code
* frameProcessors}.
*/
private final List<Size> sizes;
/**
* Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from.
*/
private final int inputExternalTexId; private final int inputExternalTexId;
/** Transformation matrix associated with the surface texture. */ /** Transformation matrix associated with the surface texture. */
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
/**
* Identifier of a framebuffer object associated with the intermediate texture that receives
* output from the prior {@link ExternalCopyFrameProcessor}, and provides input for the following
* {@link GlFrameProcessor}.
*/
private final int frameBuffer;
private final int inputWidth;
private final int inputHeight;
private final int outputWidth;
private final int outputHeight;
@Nullable private final EGLSurface debugPreviewEglSurface; @Nullable private final EGLSurface debugPreviewEglSurface;
private final int debugPreviewWidth; private final int debugPreviewWidth;
private final int debugPreviewHeight; private final int debugPreviewHeight;
...@@ -263,19 +288,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -263,19 +288,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private boolean inputStreamEnded; private boolean inputStreamEnded;
private volatile boolean releaseRequested; private volatile boolean releaseRequested;
private FrameEditor( private FrameProcessorChain(
ExecutorService singleThreadExecutorService, ExecutorService singleThreadExecutorService,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
EGLContext eglContext, EGLContext eglContext,
EGLSurface eglSurface, EGLSurface eglSurface,
ExternalCopyFrameProcessor externalCopyFrameProcessor, ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor, List<GlFrameProcessor> frameProcessors,
int inputWidth,
int inputHeight,
int inputExternalTexId, int inputExternalTexId,
int frameBuffer, int[] framebuffers,
int outputWidth, List<Size> sizes,
int outputHeight,
@Nullable EGLSurface debugPreviewEglSurface, @Nullable EGLSurface debugPreviewEglSurface,
int debugPreviewWidth, int debugPreviewWidth,
int debugPreviewHeight) { int debugPreviewHeight) {
...@@ -284,13 +306,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -284,13 +306,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.eglContext = eglContext; this.eglContext = eglContext;
this.eglSurface = eglSurface; this.eglSurface = eglSurface;
this.externalCopyFrameProcessor = externalCopyFrameProcessor; this.externalCopyFrameProcessor = externalCopyFrameProcessor;
this.transformationFrameProcessor = transformationFrameProcessor; this.frameProcessors = frameProcessors;
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
this.inputExternalTexId = inputExternalTexId; this.inputExternalTexId = inputExternalTexId;
this.frameBuffer = frameBuffer; this.framebuffers = framebuffers;
this.outputWidth = outputWidth; this.sizes = sizes;
this.outputHeight = outputHeight;
this.debugPreviewEglSurface = debugPreviewEglSurface; this.debugPreviewEglSurface = debugPreviewEglSurface;
this.debugPreviewWidth = debugPreviewWidth; this.debugPreviewWidth = debugPreviewWidth;
this.debugPreviewHeight = debugPreviewHeight; this.debugPreviewHeight = debugPreviewHeight;
...@@ -331,9 +350,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -331,9 +350,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
/** /**
* Informs the frame editor that a frame will be queued to its input surface. * Informs the {@code FrameProcessorChain} that a frame will be queued to its input surface.
* *
* <p>Should be called before rendering a frame to the frame editor's input surface. * <p>Should be called before rendering a frame to the frame processor chain's input surface.
* *
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}. * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
*/ */
...@@ -353,11 +372,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -353,11 +372,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
try { try {
oldestGlProcessingFuture.get(); oldestGlProcessingFuture.get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
throw TransformationException.createForFrameEditor( throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw TransformationException.createForFrameEditor( throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} }
oldestGlProcessingFuture = futures.peek(); oldestGlProcessingFuture = futures.peek();
...@@ -377,7 +396,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -377,7 +396,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return inputStreamEnded && !hasPendingFrames(); return inputStreamEnded && !hasPendingFrames();
} }
/** Informs the {@code FrameEditor} that no further input frames should be accepted. */ /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */
public void signalEndOfInputStream() { public void signalEndOfInputStream() {
inputStreamEnded = true; inputStreamEnded = true;
} }
...@@ -385,9 +404,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -385,9 +404,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* Releases all resources. * Releases all resources.
* *
* <p>If the frame editor is released before it has {@link #isEnded() ended}, it will attempt to * <p>If the frame processor chain is released before it has {@link #isEnded() ended}, it will
* cancel processing any input frames that have already become available. Input frames that become * attempt to cancel processing any input frames that have already become available. Input frames
* available after release are ignored. * that become available after release are ignored.
*/ */
public void release() { public void release() {
releaseRequested = true; releaseRequested = true;
...@@ -398,7 +417,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -398,7 +417,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService.submit( singleThreadExecutorService.submit(
() -> { () -> {
externalCopyFrameProcessor.release(); externalCopyFrameProcessor.release();
transformationFrameProcessor.release(); for (int i = 0; i < frameProcessors.size(); i++) {
frameProcessors.get(i).release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext); GlUtil.destroyEglContext(eglDisplay, eglContext);
})); }));
if (inputSurfaceTexture != null) { if (inputSurfaceTexture != null) {
...@@ -417,13 +438,41 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -417,13 +438,41 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
long presentationTimeNs = inputSurfaceTexture.getTimestamp(); long presentationTimeNs = inputSurfaceTexture.getTimestamp();
if (frameProcessors.isEmpty()) {
GlUtil.focusEglSurface(
eglDisplay, eglContext, eglSurface, sizes.get(0).getWidth(), sizes.get(0).getHeight());
} else {
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglContext, eglSurface, frameBuffer, inputWidth, inputHeight); eglDisplay,
eglContext,
eglSurface,
framebuffers[0],
sizes.get(0).getWidth(),
sizes.get(0).getHeight());
}
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); for (int i = 0; i < frameProcessors.size() - 1; i++) {
transformationFrameProcessor.updateProgramAndDraw(presentationTimeNs); GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i + 1],
sizes.get(i + 1).getWidth(),
sizes.get(i + 1).getHeight());
frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
}
if (!frameProcessors.isEmpty()) {
GlUtil.focusEglSurface(
eglDisplay,
eglContext,
eglSurface,
getLast(sizes).getWidth(),
getLast(sizes).getHeight());
getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs);
}
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface); EGL14.eglSwapBuffers(eglDisplay, eglSurface);
......
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
...@@ -32,6 +32,9 @@ import java.io.IOException; ...@@ -32,6 +32,9 @@ import java.io.IOException;
* </ol> * </ol>
*/ */
/* package */ interface GlFrameProcessor { /* package */ interface GlFrameProcessor {
// TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
// using a placeholder surface until the encoder surface is known. If so, convert
// configureOutputSize to a simple getter.
/** /**
* Returns the output {@link Size} of frames processed through {@link * Returns the output {@link Size} of frames processed through {@link
......
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
...@@ -27,9 +27,7 @@ import androidx.media3.common.C; ...@@ -27,9 +27,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be * Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be
...@@ -173,15 +171,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -173,15 +171,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* *
* <p>This method can only be called after {@link #configureOutputSize(int, int)}. * <p>This method can only be called after {@link #configureOutputSize(int, int)}.
*/ */
@RequiresNonNull("adjustedTransformationMatrix")
public boolean shouldProcess() { public boolean shouldProcess() {
checkStateNotNull(adjustedTransformationMatrix);
return inputWidth != outputWidth return inputWidth != outputWidth
|| inputHeight != outputHeight || inputHeight != outputHeight
|| !adjustedTransformationMatrix.isIdentity(); || !adjustedTransformationMatrix.isIdentity();
} }
@Override @Override
@EnsuresNonNull("adjustedTransformationMatrix")
public Size configureOutputSize(int inputWidth, int inputHeight) { public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth; this.inputWidth = inputWidth;
this.inputHeight = inputHeight; this.inputHeight = inputHeight;
...@@ -191,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -191,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int displayHeight = inputHeight; int displayHeight = inputHeight;
if (!transformationMatrix.isIdentity()) { if (!transformationMatrix.isIdentity()) {
float inputAspectRatio = (float) inputWidth / inputHeight; float inputAspectRatio = (float) inputWidth / inputHeight;
// Scale frames by inputAspectRatio, to account for FrameEditor's normalized device // Scale frames by inputAspectRatio, to account for FrameProcessorChain's normalized device
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular
// display of input pixels during transformations (ex. rotations). With scaling, // display of input pixels during transformations (ex. rotations). With scaling,
// transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to // transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to
......
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
...@@ -274,15 +274,15 @@ public final class TransformationException extends Exception { ...@@ -274,15 +274,15 @@ public final class TransformationException extends Exception {
} }
/** /**
* Creates an instance for a {@link FrameEditor} related exception. * Creates an instance for a {@link FrameProcessorChain} related exception.
* *
* @param cause The cause of the failure. * @param cause The cause of the failure.
* @param errorCode See {@link #errorCode}. * @param errorCode See {@link #errorCode}.
* @return The created instance. * @return The created instance.
*/ */
/* package */ static TransformationException createForFrameEditor( /* package */ static TransformationException createForFrameProcessorChain(
Throwable cause, int errorCode) { Throwable cause, int errorCode) {
return new TransformationException("FrameEditor error", cause, errorCode); return new TransformationException("FrameProcessorChain error", cause, errorCode);
} }
/** /**
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
...@@ -28,6 +28,8 @@ import androidx.annotation.RequiresApi; ...@@ -28,6 +28,8 @@ import androidx.annotation.RequiresApi;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.List; import java.util.List;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
...@@ -40,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -40,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final DecoderInputBuffer decoderInputBuffer; private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder; private final Codec decoder;
@Nullable private final FrameEditor frameEditor; @Nullable private final FrameProcessorChain frameProcessorChain;
private final Codec encoder; private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer; private final DecoderInputBuffer encoderOutputBuffer;
...@@ -74,14 +76,18 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -74,14 +76,18 @@ import org.checkerframework.dataflow.qual.Pure;
.setRotationDegrees(transformationRequest.rotationDegrees) .setRotationDegrees(transformationRequest.rotationDegrees)
.setResolution(transformationRequest.outputHeight) .setResolution(transformationRequest.outputHeight)
.build(); .build();
Size requestedEncoderDimensions = // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
scaleToFitFrameProcessor.configureOutputSize(decodedWidth, decodedHeight); ImmutableList<GlFrameProcessor> frameProcessors = ImmutableList.of(scaleToFitFrameProcessor);
List<Size> frameProcessorSizes =
FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors);
Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes);
// TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor.
outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees(); outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees();
Format requestedEncoderFormat = Format requestedEncoderFormat =
new Format.Builder() new Format.Builder()
.setWidth(requestedEncoderDimensions.getWidth()) .setWidth(requestedEncoderSize.getWidth())
.setHeight(requestedEncoderDimensions.getHeight()) .setHeight(requestedEncoderSize.getHeight())
.setRotationDegrees(0) .setRotationDegrees(0)
.setFrameRate(inputFormat.frameRate) .setFrameRate(inputFormat.frameRate)
.setSampleMimeType( .setSampleMimeType(
...@@ -104,26 +110,30 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -104,26 +110,30 @@ import org.checkerframework.dataflow.qual.Pure;
|| inputFormat.width != encoderSupportedFormat.width || inputFormat.width != encoderSupportedFormat.width
|| scaleToFitFrameProcessor.shouldProcess() || scaleToFitFrameProcessor.shouldProcess()
|| shouldAlwaysUseFrameEditor()) { || shouldAlwaysUseFrameEditor()) {
frameEditor = // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
FrameEditor.create( // has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set(
frameProcessorSizes.size() - 1,
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
frameProcessorChain =
FrameProcessorChain.create(
context, context,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
/* outputWidth= */ encoderSupportedFormat.width,
/* outputHeight= */ encoderSupportedFormat.height,
inputFormat.pixelWidthHeightRatio, inputFormat.pixelWidthHeightRatio,
scaleToFitFrameProcessor, frameProcessors,
frameProcessorSizes,
/* outputSurface= */ encoder.getInputSurface(), /* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing, transformationRequest.enableHdrEditing,
debugViewProvider); debugViewProvider);
} else { } else {
frameEditor = null; frameProcessorChain = null;
} }
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(
inputFormat, inputFormat,
frameEditor == null ? encoder.getInputSurface() : frameEditor.createInputSurface()); frameProcessorChain == null
? encoder.getInputSurface()
: frameProcessorChain.createInputSurface());
} }
@Override @Override
...@@ -139,9 +149,9 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -139,9 +149,9 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() throws TransformationException {
if (frameEditor != null) { if (frameProcessorChain != null) {
frameEditor.getAndRethrowBackgroundExceptions(); frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameEditor.isEnded()) { if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) { if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream(); encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true; signaledEndOfStreamToEncoder = true;
...@@ -163,8 +173,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -163,8 +173,8 @@ import org.checkerframework.dataflow.qual.Pure;
canProcessMoreDataImmediately = processDataDefault(); canProcessMoreDataImmediately = processDataDefault();
} }
if (decoder.isEnded()) { if (decoder.isEnded()) {
if (frameEditor != null) { if (frameProcessorChain != null) {
frameEditor.signalEndOfInputStream(); frameProcessorChain.signalEndOfInputStream();
} else { } else {
encoder.signalEndOfInputStream(); encoder.signalEndOfInputStream();
signaledEndOfStreamToEncoder = true; signaledEndOfStreamToEncoder = true;
...@@ -179,8 +189,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -179,8 +189,8 @@ import org.checkerframework.dataflow.qual.Pure;
* *
* <p>In this method the decoder could decode multiple frames in one invocation; as compared to * <p>In this method the decoder could decode multiple frames in one invocation; as compared to
* {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently, * {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently,
* if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up * if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are
* in the decoder's output surface. * queued up in the decoder's output surface.
* *
* <p>Prior to API 29, decoders may drop frames to keep their output surface from growing out of * <p>Prior to API 29, decoders may drop frames to keep their output surface from growing out of
* bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame * bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
...@@ -195,12 +205,12 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -195,12 +205,12 @@ import org.checkerframework.dataflow.qual.Pure;
/** /**
* Processes at most one input frame and returns whether a frame was processed. * Processes at most one input frame and returns whether a frame was processed.
* *
* <p>Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link * <p>Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link
* FrameEditor} has finished processing the previous frame. * FrameProcessorChain} has finished processing the previous frame.
*/ */
private boolean processDataDefault() throws TransformationException { private boolean processDataDefault() throws TransformationException {
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
if (frameEditor != null && frameEditor.hasPendingFrames()) { if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
return false; return false;
} }
return maybeProcessDecoderOutput(); return maybeProcessDecoderOutput();
...@@ -240,8 +250,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -240,8 +250,8 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public void release() { public void release() {
if (frameEditor != null) { if (frameProcessorChain != null) {
frameEditor.release(); frameProcessorChain.release();
} }
decoder.release(); decoder.release();
encoder.release(); encoder.release();
...@@ -299,8 +309,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -299,8 +309,8 @@ import org.checkerframework.dataflow.qual.Pure;
return false; return false;
} }
if (frameEditor != null) { if (frameProcessorChain != null) {
frameEditor.registerInputFrame(); frameProcessorChain.registerInputFrame();
} }
decoder.releaseOutputBuffer(/* render= */ true); decoder.releaseOutputBuffer(/* render= */ true);
return true; return true;
......
libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java 0 → 100644
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static com.google.common.truth.Truth.assertThat;
import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Robolectric tests for {@link FrameProcessorChain}.
*
* <p>See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the
* androidTest directory for instrumentation tests.
*/
@RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest {
@Test
public void configureOutputDimensions_withEmptyList_returnsInputSize() {
Size inputSize = new Size(200, 100);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of());
assertThat(sizes).containsExactly(inputSize);
}
@Test
public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() {
Size inputSize = new Size(200, 100);
Size outputSize = new Size(300, 250);
GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor));
assertThat(sizes).containsExactly(inputSize, outputSize).inOrder();
}
@Test
public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() {
Size inputSize = new Size(200, 100);
Size outputSize1 = new Size(300, 250);
Size outputSize2 = new Size(400, 244);
Size outputSize3 = new Size(150, 160);
GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1);
GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2);
GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3);
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(),
inputSize.getHeight(),
ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3));
assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder();
}
private static class FakeFrameProcessor implements GlFrameProcessor {
private final Size outputSize;
private FakeFrameProcessor(Size outputSize) {
this.outputSize = outputSize;
}
@Override
public Size configureOutputSize(int inputWidth, int inputHeight) {
return outputSize;
}
@Override
public void initialize(int inputTexId) {}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {}
@Override
public void release() {}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment