Skip to content

SDK / exoplayer

Go to a project
Commit 34f014ce by huangdarwin Committed by Ian Baker
Options

FrameProcessor: Add aspect ratio changes to Presentation. 

PiperOrigin-RevId: 441250773
parent 83daa052
Inline Side-by-side
Showing with 589 additions and 80 deletions
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java
......@@ -68,11 +68,10 @@ public final class AdvancedFrameProcessorPixelTest {
height = inputBitmap.getHeight();
// This surface is needed for focussing a render target, but the tests don't write output to it.
// The frame processor's output is written to a framebuffer instead.
EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, new SurfaceTexture(false));
EGLSurface eglSurface =
GlUtil.getEglSurface(eglDisplay, new SurfaceTexture(/* singleBufferMode= */ false));
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
inputTexId =
BitmapTestUtil.createGlTextureFromBitmap(
BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING));
inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap);
outputTexId = GlUtil.createTexture(width, height);
int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer(eglDisplay, eglContext, eglSurface, frameBuffer, width, height);
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java
......@@ -67,6 +67,21 @@ public class BitmapTestUtil {
"media/bitmap/sample_mp4_first_frame_crop_smaller.png";
public static final String CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_crop_larger.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_scale_to_fit_narrow.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_scale_to_fit_wide.png";
public static final String
ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_scale_to_fit_with_crop_narrow.png";
public static final String
ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_scale_to_fit_with_crop_wide.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_stretch_to_fit_narrow.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_aspect_ratio_stretch_to_fit_wide.png";
/**
* Maximum allowed average pixel difference between the expected and actual edited images in pixel
* difference-based tests. The value is chosen so that differences in decoder behavior across
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
......@@ -16,8 +16,6 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.BitmapTestUtil.CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
......@@ -178,10 +176,9 @@ public final class FrameProcessorChainPixelTest {
}
@Test
public void
processData_withPresentationFrameProcessor_requestOutputHeight_producesExpectedOutput()
throws Exception {
String testId = "processData_withPresentationFrameProcessor_requestOutputHeight";
public void processData_withPresentationFrameProcessor_setResolution_producesExpectedOutput()
throws Exception {
String testId = "processData_withPresentationFrameProcessor_setResolution";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext()).setResolution(480).build();
setUpAndPrepareFirstFrame(glFrameProcessor);
......@@ -200,51 +197,6 @@ public final class FrameProcessorChainPixelTest {
}
@Test
public void processData_withPresentationFrameProcessor_cropSmaller_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_cropSmaller";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
.build();
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_withPresentationFrameProcessor_cropLarger_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_cropLarger";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
.build();
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
throws Exception {
String testId = "processData_withScaleToFitFrameProcessor_rotate45";
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/PresentationFrameProcessorPixelTest.java 0 → 100644
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_SCALE_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_SCALE_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_STRETCH_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.ASPECT_RATIO_STRETCH_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING;
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLSurface;
import android.util.Size;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.GlUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Pixel test for frame processing via {@link PresentationFrameProcessor}.
*
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
* as recommended in {@link FrameProcessorChainPixelTest}.
*/
@RunWith(AndroidJUnit4.class)
public final class PresentationFrameProcessorPixelTest {
static {
GlUtil.glAssertionsEnabled = true;
}
private final EGLDisplay eglDisplay = GlUtil.createEglDisplay();
private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
private @MonotonicNonNull GlFrameProcessor presentationFrameProcessor;
private @MonotonicNonNull EGLSurface eglSurface;
private int inputTexId;
private int outputTexId;
private int inputWidth;
private int inputHeight;
@Before
public void createTextures() throws IOException {
Bitmap inputBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
inputWidth = inputBitmap.getWidth();
inputHeight = inputBitmap.getHeight();
// This surface is needed for focussing a render target, but the tests don't write output to it.
// The frame processor's output is written to a framebuffer instead.
eglSurface =
GlUtil.getEglSurface(eglDisplay, new SurfaceTexture(/* singleBufferMode= */ false));
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, inputWidth, inputHeight);
inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap);
}
@After
public void release() {
if (presentationFrameProcessor != null) {
presentationFrameProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
@Test
public void updateProgramAndDraw_noEdits_producesExpectedOutput() throws Exception {
String testId = "updateProgramAndDraw_noEdits";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext()).build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_cropSmaller_producesExpectedOutput() throws Exception {
String testId = "updateProgramAndDraw_cropSmaller";
GlFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(CROP_SMALLER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_cropLarger_producesExpectedOutput() throws Exception {
String testId = "updateProgramAndDraw_cropSmaller";
GlFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_changeAspectRatio_scaleToFit_narrow_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_scaleToFit_narrow";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(1f, PresentationFrameProcessor.SCALE_TO_FIT)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(
ASPECT_RATIO_SCALE_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_changeAspectRatio_scaleToFit_wide_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_scaleToFit_wide";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(2f, PresentationFrameProcessor.SCALE_TO_FIT)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ASPECT_RATIO_SCALE_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void
updateProgramAndDraw_changeAspectRatio_scaleToFitWithCrop_narrow_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_scaleToFitWithCrop_narrow";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(1f, PresentationFrameProcessor.SCALE_TO_FIT_WITH_CROP)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(
ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void
updateProgramAndDraw_changeAspectRatio_scaleToFitWithCrop_wide_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_scaleToFitWithCrop_wide";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(2f, PresentationFrameProcessor.SCALE_TO_FIT_WITH_CROP)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(
ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_changeAspectRatio_stretchToFit_narrow_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_stretchToFit_narrow";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(1f, PresentationFrameProcessor.STRETCH_TO_FIT)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(
ASPECT_RATIO_STRETCH_TO_FIT_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void updateProgramAndDraw_changeAspectRatio_stretchToFit_wide_producesExpectedOutput()
throws Exception {
String testId = "updateProgramAndDraw_changeAspectRatio_stretchToFit_wide";
presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(2f, PresentationFrameProcessor.STRETCH_TO_FIT)
.build();
presentationFrameProcessor.initialize(inputTexId, inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(
ASPECT_RATIO_STRETCH_TO_FIT_WIDE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
presentationFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.getWidth(), outputSize.getHeight());
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private void setupOutputTexture(int outputWidth, int outputHeight) throws IOException {
outputTexId = GlUtil.createTexture(outputWidth, outputHeight);
int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
Assertions.checkNotNull(eglSurface),
frameBuffer,
outputWidth,
outputHeight);
}
}
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
......@@ -18,28 +18,56 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static java.lang.annotation.ElementType.TYPE_USE;
import static java.lang.annotation.RetentionPolicy.SOURCE;
import android.content.Context;
import android.graphics.Matrix;
import android.util.Size;
import androidx.annotation.IntDef;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Controls how a frame is viewed, by cropping or changing resolution.
* Controls how a frame is viewed, by cropping, changing aspect ratio, or changing resolution.
*
* <p>Cropping is applied before setting resolution.
* <p>Cropping or aspect ratio is applied before setting resolution.
*/
// TODO(b/213190310): Implement aspect ratio changes, etc.
@UnstableApi
public final class PresentationFrameProcessor implements GlFrameProcessor {
/**
* Strategies for how to apply the presented frame. One of {@link #SCALE_TO_FIT}, {@link
* #SCALE_TO_FIT_WITH_CROP}, or {@link #STRETCH_TO_FIT}.
*/
@Documented
@Retention(SOURCE)
@Target(TYPE_USE)
@IntDef({SCALE_TO_FIT, SCALE_TO_FIT_WITH_CROP, STRETCH_TO_FIT})
public @interface PresentationStrategy {}
/**
* Empty pixels added above and below the input frame (for letterboxing), or to the left and right
* of the input frame (for pillarboxing), until the desired aspect ratio is achieved. All input
* frame pixels will be within the output frame.
*/
public static final int SCALE_TO_FIT = 0;
/**
* Pixels cropped from the input frame, until the desired aspect ratio is achieved. Pixels will be
* cropped either from the top and bottom, or from the left and right sides, of the input frame.
*/
public static final int SCALE_TO_FIT_WITH_CROP = 1;
/** Frame stretched larger on the x or y axes to fit the desired aspect ratio. */
public static final int STRETCH_TO_FIT = 2;
/** A builder for {@link PresentationFrameProcessor} instances. */
public static final class Builder {
......@@ -52,6 +80,8 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
private float cropRight;
private float cropBottom;
private float cropTop;
private float aspectRatio;
private @PresentationStrategy int presentationStrategy;
/**
* Creates a builder with default values.
......@@ -65,6 +95,7 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
cropRight = 1f;
cropBottom = -1f;
cropTop = 1f;
aspectRatio = C.LENGTH_UNSET;
}
/**
......@@ -95,6 +126,9 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
*
* <p>Width and height values set may be rescaled by {@link #setResolution(int)}.
*
* <p>Only one of {@code setCrop} or {@link #setAspectRatio(float, int)} can be called for one
* {@link PresentationFrameProcessor}.
*
* @param left The left edge of the output frame, in NDC. Must be less than {@code right}.
* @param right The right edge of the output frame, in NDC. Must be greater than {@code left}.
* @param bottom The bottom edge of the output frame, in NDC. Must be less than {@code top}.
......@@ -106,6 +140,9 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
right > left, "right value " + right + " should be greater than left value " + left);
checkArgument(
top > bottom, "top value " + top + " should be greater than bottom value " + bottom);
checkState(
aspectRatio == C.LENGTH_UNSET,
"setAspectRatio and setCrop cannot be called in the same instance");
cropLeft = left;
cropRight = right;
cropBottom = bottom;
......@@ -114,9 +151,45 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
return this;
}
/**
* Resize a frame's width or height to conform to an {@code aspectRatio}, given a {@link
* PresentationStrategy}, and leaving input pixels unchanged.
*
* <p>Width and height values set here may be rescaled by {@link #setResolution(int)}.
*
* <p>Only one of {@link #setCrop(float, float, float, float)} or {@code setAspectRatio} can be
* called for one {@link PresentationFrameProcessor}.
*
* @param aspectRatio The aspect ratio of the output frame, defined as width/height. Must be
* positive.
* @return This builder.
*/
public Builder setAspectRatio(
float aspectRatio, @PresentationStrategy int presentationStrategy) {
checkArgument(aspectRatio > 0, "aspect ratio " + aspectRatio + " must be positive");
checkArgument(
presentationStrategy == SCALE_TO_FIT
|| presentationStrategy == SCALE_TO_FIT_WITH_CROP
|| presentationStrategy == STRETCH_TO_FIT,
"invalid presentationStrategy " + presentationStrategy);
checkState(
cropLeft == -1f && cropRight == 1f && cropBottom == -1f && cropTop == 1f,
"setAspectRatio and setCrop cannot be called in the same instance");
this.aspectRatio = aspectRatio;
this.presentationStrategy = presentationStrategy;
return this;
}
public PresentationFrameProcessor build() {
return new PresentationFrameProcessor(
context, heightPixels, cropLeft, cropRight, cropBottom, cropTop);
context,
heightPixels,
cropLeft,
cropRight,
cropBottom,
cropTop,
aspectRatio,
presentationStrategy);
}
}
......@@ -130,9 +203,12 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
private final float cropRight;
private final float cropBottom;
private final float cropTop;
private final float requestedAspectRatio;
private final @PresentationStrategy int presentationStrategy;
private int outputRotationDegrees;
private @MonotonicNonNull Size outputSize;
private int outputWidth;
private int outputHeight;
private @MonotonicNonNull Matrix transformationMatrix;
private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
......@@ -143,14 +219,20 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
float cropLeft,
float cropRight,
float cropBottom,
float cropTop) {
float cropTop,
float requestedAspectRatio,
@PresentationStrategy int presentationStrategy) {
this.context = context;
this.requestedHeightPixels = requestedHeightPixels;
this.cropLeft = cropLeft;
this.cropRight = cropRight;
this.cropBottom = cropBottom;
this.cropTop = cropTop;
this.requestedAspectRatio = requestedAspectRatio;
this.presentationStrategy = presentationStrategy;
outputWidth = C.LENGTH_UNSET;
outputHeight = C.LENGTH_UNSET;
outputRotationDegrees = C.LENGTH_UNSET;
transformationMatrix = new Matrix();
}
......@@ -164,7 +246,10 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
checkState(
outputWidth != C.LENGTH_UNSET && outputHeight != C.LENGTH_UNSET,
"configureOutputSizeAndTransformationMatrix must be called before getOutputSize");
return new Size(outputWidth, outputHeight);
}
/**
......@@ -175,7 +260,10 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
* <p>The frame processor must be {@linkplain #initialize(int,int,int) initialized}.
*/
public int getOutputRotationDegrees() {
checkState(outputRotationDegrees != C.LENGTH_UNSET);
checkState(
outputRotationDegrees != C.LENGTH_UNSET,
"configureOutputSizeAndTransformationMatrix must be called before"
+ " getOutputRotationDegrees");
return outputRotationDegrees;
}
......@@ -194,35 +282,44 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
@EnsuresNonNull("transformationMatrix")
@VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL.
/* package */ void configureOutputSizeAndTransformationMatrix(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
transformationMatrix = new Matrix();
outputWidth = inputWidth;
outputHeight = inputHeight;
Size cropSize = applyCrop(inputWidth, inputHeight);
int displayWidth = cropSize.getWidth();
int displayHeight = cropSize.getHeight();
if (cropLeft != -1f || cropRight != 1f || cropBottom != -1f || cropTop != 1f) {
checkState(
requestedAspectRatio == C.LENGTH_UNSET,
"aspect ratio and crop cannot both be set in the same instance");
applyCrop();
} else if (requestedAspectRatio != C.LENGTH_UNSET) {
applyAspectRatio();
}
// Scale width and height to desired requestedHeightPixels, preserving aspect ratio.
if (requestedHeightPixels != C.LENGTH_UNSET && requestedHeightPixels != displayHeight) {
displayWidth = Math.round((float) requestedHeightPixels * displayWidth / displayHeight);
displayHeight = requestedHeightPixels;
if (requestedHeightPixels != C.LENGTH_UNSET && requestedHeightPixels != outputHeight) {
outputWidth = Math.round((float) requestedHeightPixels * outputWidth / outputHeight);
outputHeight = requestedHeightPixels;
}
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// frame before encoding, so the encoded frame's width >= height, and set
// outputRotationDegrees to ensure the frame is displayed in the correct orientation.
if (displayHeight > displayWidth) {
if (outputHeight > outputWidth) {
outputRotationDegrees = 90;
// TODO(b/201293185): After fragment shader transformations are implemented, put
// postRotate in a later GlFrameProcessor.
// TODO(b/201293185): Put postRotate in a later GlFrameProcessor.
transformationMatrix.postRotate(outputRotationDegrees);
outputSize = new Size(displayHeight, displayWidth);
int swap = outputWidth;
outputWidth = outputHeight;
outputHeight = swap;
} else {
outputRotationDegrees = 0;
outputSize = new Size(displayWidth, displayHeight);
}
}
@RequiresNonNull("transformationMatrix")
private Size applyCrop(int inputWidth, int inputHeight) {
private void applyCrop() {
float scaleX = (cropRight - cropLeft) / GlUtil.LENGTH_NDC;
float scaleY = (cropTop - cropBottom) / GlUtil.LENGTH_NDC;
float centerX = (cropLeft + cropRight) / 2;
......@@ -231,8 +328,35 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
transformationMatrix.postTranslate(-centerX, -centerY);
transformationMatrix.postScale(1f / scaleX, 1f / scaleY);
int outputWidth = Math.round(inputWidth * scaleX);
int outputHeight = Math.round(inputHeight * scaleY);
return new Size(outputWidth, outputHeight);
outputWidth = Math.round(outputWidth * scaleX);
outputHeight = Math.round(outputHeight * scaleY);
}
@RequiresNonNull("transformationMatrix")
private void applyAspectRatio() {
float inputAspectRatio = (float) outputWidth / outputHeight;
if (presentationStrategy == SCALE_TO_FIT) {
if (requestedAspectRatio > inputAspectRatio) {
transformationMatrix.setScale(inputAspectRatio / requestedAspectRatio, 1f);
outputWidth = Math.round(outputHeight * requestedAspectRatio);
} else {
transformationMatrix.setScale(1f, requestedAspectRatio / inputAspectRatio);
outputHeight = Math.round(outputWidth / requestedAspectRatio);
}
} else if (presentationStrategy == SCALE_TO_FIT_WITH_CROP) {
if (requestedAspectRatio > inputAspectRatio) {
transformationMatrix.setScale(1f, requestedAspectRatio / inputAspectRatio);
outputHeight = Math.round(outputWidth / requestedAspectRatio);
} else {
transformationMatrix.setScale(inputAspectRatio / requestedAspectRatio, 1f);
outputWidth = Math.round(outputHeight * requestedAspectRatio);
}
} else if (presentationStrategy == STRETCH_TO_FIT) {
if (requestedAspectRatio > inputAspectRatio) {
outputWidth = Math.round(outputHeight * requestedAspectRatio);
} else {
outputHeight = Math.round(outputWidth / requestedAspectRatio);
}
}
}
}
libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java
......@@ -131,7 +131,7 @@ public final class PresentationFrameProcessorTest {
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(left, right, bottom, top)
.setResolution(100)
.setResolution(requestedHeight)
.build();
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
......@@ -157,7 +157,7 @@ public final class PresentationFrameProcessorTest {
int requestedHeight = 100;
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setResolution(100)
.setResolution(requestedHeight)
.setCrop(left, right, bottom, top)
.build();
......@@ -174,6 +174,70 @@ public final class PresentationFrameProcessorTest {
}
@Test
public void getOutputSize_setAspectRatio_changesDimensions() {
int inputWidth = 300;
int inputHeight = 200;
float aspectRatio = 2f;
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(aspectRatio, PresentationFrameProcessor.SCALE_TO_FIT)
.build();
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@Test
public void getOutputSize_setAspectRatioAndResolution_changesDimensions() {
int inputWidth = 300;
int inputHeight = 200;
float aspectRatio = 2f;
int requestedHeight = 100;
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(aspectRatio, PresentationFrameProcessor.SCALE_TO_FIT)
.setResolution(requestedHeight)
.build();
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * requestedHeight));
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}
@Test
public void getOutputSize_setAspectRatioAndCrop_throwsIllegalStateException() {
PresentationFrameProcessor.Builder presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setAspectRatio(/* aspectRatio= */ 2f, PresentationFrameProcessor.SCALE_TO_FIT);
assertThrows(
IllegalStateException.class,
() ->
presentationFrameProcessor.setCrop(
/* left= */ -.5f, /* right= */ .5f, /* bottom= */ .5f, /* top= */ 1f));
}
@Test
public void getOutputSize_setCropAndAspectRatio_throwsIllegalStateException() {
PresentationFrameProcessor.Builder presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext())
.setCrop(/* left= */ -.5f, /* right= */ .5f, /* bottom= */ .5f, /* top= */ 1f);
assertThrows(
IllegalStateException.class,
() ->
presentationFrameProcessor.setAspectRatio(
/* aspectRatio= */ 2f, PresentationFrameProcessor.SCALE_TO_FIT));
}
@Test
public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder(getApplicationContext()).build();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment