Skip to content

SDK / exoplayer

Go to a project
Commit 47f4d905 by hschlueter Committed by tonihei
Options

Use TransformationException for GL errors. 

PiperOrigin-RevId: 418820557
parent a9edb207
Inline Side-by-side
Showing with 373 additions and 265 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java 0 → 100644
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertThat;
import static java.lang.Math.abs;
import static java.lang.Math.max;
import android.content.res.AssetFileDescriptor;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Matrix;
import android.graphics.PixelFormat;
import android.media.Image;
import android.media.ImageReader;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import androidx.annotation.Nullable;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.util.MimeTypes;
import java.io.InputStream;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Test for frame processing via {@link FrameEditor#processData()}. */
@RunWith(AndroidJUnit4.class)
public final class FrameEditorDataProcessingTest {
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame.png";
/**
* Maximum allowed average pixel difference between the expected and actual edited images for the
* test to pass. The value is chosen so that differences in decoder behavior across emulator
* versions shouldn't affect whether the test passes, but substantial distortions introduced by
* changes in the behavior of the frame editor will cause the test to fail.
*/
private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
/** Timeout for dequeueing buffers from the codec, in microseconds. */
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
/** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */
private static final int SURFACE_WAIT_MS = 1000;
/** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
private @MonotonicNonNull FrameEditor frameEditor;
private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat;
@Before
public void setUp() throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
try (AssetFileDescriptor afd =
getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) {
mediaFormat = mediaExtractor.getTrackFormat(i);
mediaExtractor.selectTrack(i);
break;
}
}
int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
frameEditorOutputImageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Matrix identityMatrix = new Matrix();
frameEditor =
FrameEditor.create(
getApplicationContext(),
width,
height,
PIXEL_WIDTH_HEIGHT_RATIO,
identityMatrix,
frameEditorOutputImageReader.getSurface(),
Transformer.DebugViewProvider.NONE);
// Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure(
mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]);
int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0);
mediaCodec.queueInputBuffer(
inputBufferIndex,
/* offset= */ 0,
sampleSize,
mediaExtractor.getSampleTime(),
mediaExtractor.getSampleFlags());
// Queue an end-of-stream buffer to force the codec to produce output.
inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
mediaCodec.queueInputBuffer(
inputBufferIndex,
/* offset= */ 0,
/* size= */ 0,
/* presentationTimeUs= */ 0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
// Dequeue and render the output video frame.
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
int outputBufferIndex;
do {
outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US);
assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
} while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
|| outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
// Sleep to give time for the surface texture to be populated.
Thread.sleep(SURFACE_WAIT_MS);
assertThat(frameEditor.hasInputData()).isTrue();
} finally {
mediaExtractor.release();
if (mediaCodec != null) {
mediaCodec.release();
}
}
}
@After
public void tearDown() {
if (frameEditor != null) {
frameEditor.release();
}
}
@Test
public void processData_noEdits_producesExpectedOutput() throws Exception {
Bitmap expectedBitmap;
try (InputStream inputStream =
getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) {
expectedBitmap = BitmapFactory.decodeStream(inputStream);
}
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
// TODO(internal b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
/**
* Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per
* component image.
*/
private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) {
int width = image.getWidth();
int height = image.getHeight();
assertThat(image.getPlanes()).hasLength(1);
assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888);
Image.Plane plane = image.getPlanes()[0];
ByteBuffer buffer = plane.getBuffer();
int[] colors = new int[width * height];
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int offset = y * plane.getRowStride() + x * plane.getPixelStride();
int r = buffer.get(offset) & 0xFF;
int g = buffer.get(offset + 1) & 0xFF;
int b = buffer.get(offset + 2) & 0xFF;
int a = buffer.get(offset + 3) & 0xFF;
colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b;
}
}
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
}
/**
* Returns the sum of the absolute differences between the expected and actual bitmaps, calculated
* using the maximum difference across all color channels for each pixel, then divided by the
* total number of pixels in the image. The bitmap resolutions must match and they must use
* configuration {@link Bitmap.Config#ARGB_8888}.
*/
private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) {
int width = actual.getWidth();
int height = actual.getHeight();
assertThat(width).isEqualTo(expected.getWidth());
assertThat(height).isEqualTo(expected.getHeight());
assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888);
long sumMaximumAbsoluteDifferences = 0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int color = actual.getPixel(x, y);
int expectedColor = expected.getPixel(x, y);
int maximumAbsoluteDifference = 0;
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF)));
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF)));
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF)));
maximumAbsoluteDifference =
max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF)));
sumMaximumAbsoluteDifferences += maximumAbsoluteDifference;
}
}
return (float) sumMaximumAbsoluteDifferences / (width * height);
}
}
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java
...@@ -16,221 +16,55 @@ ...@@ -16,221 +16,55 @@
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static java.lang.Math.abs; import static org.junit.Assert.assertThrows;
import static java.lang.Math.max;
import android.content.res.AssetFileDescriptor; import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Matrix; import android.graphics.Matrix;
import android.graphics.PixelFormat; import android.graphics.SurfaceTexture;
import android.media.Image; import android.view.Surface;
import android.media.ImageReader;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import androidx.annotation.Nullable;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.util.MimeTypes;
import java.io.InputStream;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
/** Test for frame processing via {@link FrameEditor}. */ /**
* Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface,
* Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
*/
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameEditorTest { public final class FrameEditorTest {
// TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame.png";
/**
* Maximum allowed average pixel difference between the expected and actual edited images for the
* test to pass. The value is chosen so that differences in decoder behavior across emulator
* versions shouldn't affect whether the test passes, but substantial distortions introduced by
* changes in the behavior of the frame editor will cause the test to fail.
*/
private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
/** Timeout for dequeueing buffers from the codec, in microseconds. */
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
/** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */
private static final int SURFACE_WAIT_MS = 1000;
/** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
private @MonotonicNonNull FrameEditor frameEditor;
private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat;
@Before
public void setUp() throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
try (AssetFileDescriptor afd =
getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) {
mediaFormat = mediaExtractor.getTrackFormat(i);
mediaExtractor.selectTrack(i);
break;
}
}
int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
frameEditorOutputImageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Matrix identityMatrix = new Matrix();
frameEditor =
FrameEditor.create(
getApplicationContext(),
width,
height,
PIXEL_WIDTH_HEIGHT_RATIO,
identityMatrix,
frameEditorOutputImageReader.getSurface(),
Transformer.DebugViewProvider.NONE);
// Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure(
mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]);
int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0);
mediaCodec.queueInputBuffer(
inputBufferIndex,
/* offset= */ 0,
sampleSize,
mediaExtractor.getSampleTime(),
mediaExtractor.getSampleFlags());
// Queue an end-of-stream buffer to force the codec to produce output.
inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
mediaCodec.queueInputBuffer(
inputBufferIndex,
/* offset= */ 0,
/* size= */ 0,
/* presentationTimeUs= */ 0,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
// Dequeue and render the output video frame.
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
int outputBufferIndex;
do {
outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US);
assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
} while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
|| outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
// Sleep to give time for the surface texture to be populated.
Thread.sleep(SURFACE_WAIT_MS);
assertThat(frameEditor.hasInputData()).isTrue();
} finally {
mediaExtractor.release();
if (mediaCodec != null) {
mediaCodec.release();
}
}
}
@After
public void tearDown() {
if (frameEditor != null) {
frameEditor.release();
}
}
@Test @Test
public void processData_noEdits_producesExpectedOutput() throws Exception { public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
Bitmap expectedBitmap; throws TransformationException {
try (InputStream inputStream = FrameEditor.create(
getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) { getApplicationContext(),
expectedBitmap = BitmapFactory.decodeStream(inputStream); /* outputWidth= */ 200,
} /* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1,
checkNotNull(frameEditor).processData(); new Matrix(),
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); new Surface(new SurfaceTexture(false)),
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); Transformer.DebugViewProvider.NONE);
// TODO(internal b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
/**
* Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per
* component image.
*/
private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) {
int width = image.getWidth();
int height = image.getHeight();
assertThat(image.getPlanes()).hasLength(1);
assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888);
Image.Plane plane = image.getPlanes()[0];
ByteBuffer buffer = plane.getBuffer();
int[] colors = new int[width * height];
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int offset = y * plane.getRowStride() + x * plane.getPixelStride();
int r = buffer.get(offset) & 0xFF;
int g = buffer.get(offset + 1) & 0xFF;
int b = buffer.get(offset + 2) & 0xFF;
int a = buffer.get(offset + 3) & 0xFF;
colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b;
}
}
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
} }
/** @Test
* Returns the sum of the absolute differences between the expected and actual bitmaps, calculated public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
* using the maximum difference across all color channels for each pixel, then divided by the TransformationException exception =
* total number of pixels in the image. The bitmap resolutions must match and they must use assertThrows(
* configuration {@link Bitmap.Config#ARGB_8888}. TransformationException.class,
*/ () ->
private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) { FrameEditor.create(
int width = actual.getWidth(); getApplicationContext(),
int height = actual.getHeight(); /* outputWidth= */ 200,
assertThat(width).isEqualTo(expected.getWidth()); /* outputHeight= */ 100,
assertThat(height).isEqualTo(expected.getHeight()); /* pixelWidthHeightRatio= */ 2,
assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888); new Matrix(),
long sumMaximumAbsoluteDifferences = 0; new Surface(new SurfaceTexture(false)),
for (int y = 0; y < height; y++) { Transformer.DebugViewProvider.NONE));
for (int x = 0; x < width; x++) {
int color = actual.getPixel(x, y); assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
int expectedColor = expected.getPixel(x, y); assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
int maximumAbsoluteDifference = 0;
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF)));
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF)));
maximumAbsoluteDifference =
max(
maximumAbsoluteDifference,
abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF)));
maximumAbsoluteDifference =
max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF)));
sumMaximumAbsoluteDifferences += maximumAbsoluteDifference;
}
}
return (float) sumMaximumAbsoluteDifferences / (width * height);
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java
...@@ -52,6 +52,9 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -52,6 +52,9 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param outputSurface The {@link Surface}. * @param outputSurface The {@link Surface}.
* @param debugViewProvider Provider for optional debug views to show intermediate output. * @param debugViewProvider Provider for optional debug views to show intermediate output.
* @return A configured {@code FrameEditor}. * @return A configured {@code FrameEditor}.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components.
*/ */
public static FrameEditor create( public static FrameEditor create(
Context context, Context context,
...@@ -63,30 +66,71 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -63,30 +66,71 @@ import java.util.concurrent.atomic.AtomicInteger;
Transformer.DebugViewProvider debugViewProvider) Transformer.DebugViewProvider debugViewProvider)
throws TransformationException { throws TransformationException {
if (pixelWidthHeightRatio != 1.0f) { if (pixelWidthHeightRatio != 1.0f) {
// TODO(http://b/211782176): Consider implementing support for non-square pixels. // TODO(b/211782176): Consider implementing support for non-square pixels.
throw new TransformationException( throw TransformationException.createForFrameEditor(
"FrameEditor Error", new UnsupportedOperationException(
new IllegalArgumentException(
"Transformer's frame editor currently does not support frame edits on non-square" "Transformer's frame editor currently does not support frame edits on non-square"
+ " pixels. The pixelWidthHeightRatio is: " + " pixels. The pixelWidthHeightRatio is: "
+ pixelWidthHeightRatio), + pixelWidthHeightRatio),
TransformationException.ERROR_CODE_GL_INIT_FAILED); TransformationException.ERROR_CODE_GL_INIT_FAILED);
} }
EGLDisplay eglDisplay = GlUtil.createEglDisplay(); @Nullable
EGLContext eglContext = GlUtil.createEglContext(eglDisplay); SurfaceView debugSurfaceView =
EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
int textureId = GlUtil.createExternalTexture(); EGLDisplay eglDisplay;
EGLContext eglContext;
EGLSurface eglSurface;
int textureId;
GlUtil.Program glProgram; GlUtil.Program glProgram;
@Nullable EGLSurface debugPreviewEglSurface;
try { try {
// TODO(internal b/205002913): check the loaded program is consistent with the attributes eglDisplay = GlUtil.createEglDisplay();
// and uniforms expected in the code. eglContext = GlUtil.createEglContext(eglDisplay);
glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
} catch (IOException e) { GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
throw new IllegalStateException(e); textureId = GlUtil.createExternalTexture();
glProgram = configureGlProgram(context, transformationMatrix, textureId);
debugPreviewEglSurface =
debugSurfaceView == null
? null
: GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
} catch (IOException | GlUtil.GlException e) {
throw TransformationException.createForFrameEditor(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
int debugPreviewWidth;
int debugPreviewHeight;
if (debugSurfaceView != null) {
debugPreviewWidth = debugSurfaceView.getWidth();
debugPreviewHeight = debugSurfaceView.getHeight();
} else {
debugPreviewWidth = C.LENGTH_UNSET;
debugPreviewHeight = C.LENGTH_UNSET;
} }
return new FrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
glProgram,
outputWidth,
outputHeight,
debugPreviewEglSurface,
debugPreviewWidth,
debugPreviewHeight);
}
private static GlUtil.Program configureGlProgram(
Context context, Matrix transformationMatrix, int textureId) throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
GlUtil.Program glProgram =
new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
glProgram.setBufferAttribute( glProgram.setBufferAttribute(
"aPosition", "aPosition",
new float[] { new float[] {
...@@ -109,34 +153,7 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -109,34 +153,7 @@ import java.util.concurrent.atomic.AtomicInteger;
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix); float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray); glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray);
return glProgram;
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
@Nullable EGLSurface debugPreviewEglSurface;
int debugPreviewWidth;
int debugPreviewHeight;
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
debugPreviewWidth = debugSurfaceView.getWidth();
debugPreviewHeight = debugSurfaceView.getHeight();
} else {
debugPreviewEglSurface = null;
debugPreviewWidth = C.LENGTH_UNSET;
debugPreviewHeight = C.LENGTH_UNSET;
}
return new FrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
glProgram,
outputWidth,
outputHeight,
debugPreviewEglSurface,
debugPreviewWidth,
debugPreviewHeight);
} }
/** /**
...@@ -240,22 +257,31 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -240,22 +257,31 @@ import java.util.concurrent.atomic.AtomicInteger;
return pendingInputFrameCount.get() > 0; return pendingInputFrameCount.get() > 0;
} }
/** Processes pending input frame. */ /**
public void processData() { * Processes pending input frame.
inputSurfaceTexture.updateTexImage(); *
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); * @throws TransformationException If an OpenGL error occurs while processing the data.
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); */
glProgram.bindAttributesAndUniforms(); public void processData() throws TransformationException {
try {
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
glProgram.bindAttributesAndUniforms();
focusAndDrawQuad(eglSurface, outputWidth, outputHeight); focusAndDrawQuad(eglSurface, outputWidth, outputHeight);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface); EGL14.eglSwapBuffers(eglDisplay, eglSurface);
pendingInputFrameCount.decrementAndGet(); pendingInputFrameCount.decrementAndGet();
if (debugPreviewEglSurface != null) { if (debugPreviewEglSurface != null) {
focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
} catch (GlUtil.GlException e) {
throw TransformationException.createForFrameEditor(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} }
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java
...@@ -229,7 +229,7 @@ public final class TransformationException extends Exception { ...@@ -229,7 +229,7 @@ public final class TransformationException extends Exception {
} }
/** /**
* Creates an instance for an audio processing related exception. * Creates an instance for an {@link AudioProcessor} related exception.
* *
* @param cause The cause of the failure. * @param cause The cause of the failure.
* @param componentName The name of the {@link AudioProcessor} used. * @param componentName The name of the {@link AudioProcessor} used.
...@@ -244,6 +244,18 @@ public final class TransformationException extends Exception { ...@@ -244,6 +244,18 @@ public final class TransformationException extends Exception {
} }
/** /**
* Creates an instance for a {@link FrameEditor} related exception.
*
* @param cause The cause of the failure.
* @param errorCode See {@link #errorCode}.
* @return The created instance.
*/
/* package */ static TransformationException createForFrameEditor(
Throwable cause, int errorCode) {
return new TransformationException("FrameEditor error", cause, errorCode);
}
/**
* Creates an instance for a muxer related exception. * Creates an instance for a muxer related exception.
* *
* @param cause The cause of the failure. * @param cause The cause of the failure.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java
...@@ -81,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -81,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
// The decoder rotates videos to their intended display orientation. The frameEditor rotates // The decoder rotates videos to their intended display orientation. The frameEditor rotates
// them back for improved encoder compatibility. // them back for improved encoder compatibility.
// TODO(internal b/201293185): After fragment shader transformations are implemented, put // TODO(b/201293185): After fragment shader transformations are implemented, put
// postrotation in a later vertex shader. // postrotation in a later vertex shader.
transformationRequest.transformationMatrix.postRotate(outputRotationDegrees); transformationRequest.transformationMatrix.postRotate(outputRotationDegrees);
...@@ -129,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -129,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public boolean processData() { public boolean processData() throws TransformationException {
if (decoder.isEnded()) { if (decoder.isEnded()) {
return false; return false;
} }
...@@ -155,7 +155,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -155,7 +155,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Transformer}, using this method requires API level 29 or higher. * Transformer}, using this method requires API level 29 or higher.
*/ */
@RequiresApi(29) @RequiresApi(29)
private boolean processDataV29() { private boolean processDataV29() throws TransformationException {
if (frameEditor != null) { if (frameEditor != null) {
while (frameEditor.hasInputData()) { while (frameEditor.hasInputData()) {
// Processes as much frames in one invocation: FrameEditor's output surface will block // Processes as much frames in one invocation: FrameEditor's output surface will block
...@@ -170,7 +170,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -170,7 +170,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
if (decoder.isEnded()) { if (decoder.isEnded()) {
// TODO(internal b/208986865): Handle possible last frame drop. // TODO(b/208986865): Handle possible last frame drop.
encoder.signalEndOfInputStream(); encoder.signalEndOfInputStream();
return false; return false;
} }
...@@ -179,7 +179,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -179,7 +179,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** Processes input data. */ /** Processes input data. */
private boolean processDataDefault() { private boolean processDataDefault() throws TransformationException {
if (frameEditor != null) { if (frameEditor != null) {
if (frameEditor.hasInputData()) { if (frameEditor.hasInputData()) {
waitingForFrameEditorInput = false; waitingForFrameEditorInput = false;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment