Skip to content

SDK / exoplayer

Go to a project
Commit 19e21d6f by hschlueter Committed by Ian Baker
Options

Split out ExternalCopyFrameProcessor. 

All (later customizable) GlFrameProcessors after the
ExternalCopyFrameProcessor receive their input from a normal OpenGL
texture not an external texture, so they won't need to worry about
the textureTransformMatrix.

PiperOrigin-RevId: 430165652
parent 25e58f5f
Inline Side-by-side
Showing with 383 additions and 185 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java
......@@ -116,8 +116,8 @@ public final class FrameEditorDataProcessingTest {
public void processData_noEdits_producesExpectedOutput() throws Exception {
Matrix identityMatrix = new Matrix();
setUpAndPrepareFirstFrame(identityMatrix);
Bitmap expectedBitmap = getBitmap(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
......@@ -134,8 +134,8 @@ public final class FrameEditorDataProcessingTest {
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
setUpAndPrepareFirstFrame(translateRightMatrix);
Bitmap expectedBitmap = getBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
......@@ -174,8 +174,8 @@ public final class FrameEditorDataProcessingTest {
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
setUpAndPrepareFirstFrame(rotate90Matrix);
Bitmap expectedBitmap = getBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
checkNotNull(frameEditor).processData();
Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
......@@ -206,13 +206,14 @@ public final class FrameEditorDataProcessingTest {
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
frameEditorOutputImageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Context context = getApplicationContext();
frameEditor =
FrameEditor.create(
getApplicationContext(),
context,
width,
height,
PIXEL_WIDTH_HEIGHT_RATIO,
transformationMatrix,
new TransformationFrameProcessor(context, transformationMatrix),
frameEditorOutputImageReader.getSurface(),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java
......@@ -28,7 +28,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface, boolean,
* Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean,
* Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
*/
@RunWith(AndroidJUnit4.class)
......@@ -39,12 +39,14 @@ public final class FrameEditorTest {
@Test
public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
Context context = getApplicationContext();
FrameEditor.create(
getApplicationContext(),
context,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1,
new Matrix(),
new TransformationFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
......@@ -52,16 +54,18 @@ public final class FrameEditorTest {
@Test
public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
Context context = getApplicationContext();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
FrameEditor.create(
getApplicationContext(),
context,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 2,
new Matrix(),
new TransformationFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE));
......
library/transformer/src/main/assets/shaders/fragment_shader_copy_es2.glsl 0 → 100644
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler,
// copying from this texture to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
varying vec2 vTexCoords;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexCoords);
}
library/transformer/src/main/assets/shaders/fragment_shader_copy_external.glsl → library/transformer/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
......@@ -12,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Fragment shader that samples from an external texture with uTexSampler,
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output.
#extension GL_OES_EGL_image_external : require
......
library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl 0 → 100644
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
attribute vec4 aFramePosition;
attribute vec4 aTexCoords;
uniform mat4 uTexTransform;
varying vec2 vTexCoords;
void main() {
gl_Position = aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
}
library/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl → library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl
......@@ -13,15 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 vertex shader that applies the 4 * 4 transformation matrix
// uTransformationMatrix.
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
in vec4 aFramePosition;
in vec4 aTexCoords;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
out vec2 vTexCoords;
void main() {
gl_Position = uTransformationMatrix * aFramePosition;
gl_Position = aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
}
library/transformer/src/main/assets/shaders/vertex_shader_transformation.glsl → library/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
......@@ -12,15 +13,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Vertex shader that applies the 4 * 4 transformation matrix
// ES 2 vertex shader that applies the 4 * 4 transformation matrix
// uTransformationMatrix.
attribute vec4 aFramePosition;
attribute vec4 aTexCoords;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
varying vec2 vTexCoords;
void main() {
gl_Position = uTransformationMatrix * aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
vTexCoords = aTexCoords.xy;
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalCopyFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
"shaders/vertex_shader_tex_transform_es2.glsl";
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
"shaders/vertex_shader_tex_transform_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final Context context;
private final boolean enableExperimentalHdrEditing;
private @MonotonicNonNull GlProgram glProgram;
public ExternalCopyFrameProcessor(Context context, boolean enableExperimentalHdrEditing) {
this.context = context;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
@Override
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code.
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
: VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
if (enableExperimentalHdrEditing) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
}
/**
* Sets the texture transform matrix for converting an external surface texture's coordinates to
* sampling locations.
*
* @param textureTransformMatrix The external surface texture's {@link
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
@Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java
......@@ -19,7 +19,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
......@@ -40,7 +39,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* <p>Input becomes available on its {@link #getInputSurface() input surface} asynchronously so
* {@link #canProcessData()} needs to be checked before calling {@link #processData()}. Output is
* written to its {@link #create(Context, int, int, float, Matrix, Surface, boolean,
* written to its {@link #create(Context, int, int, float, GlFrameProcessor, Surface, boolean,
* Transformer.DebugViewProvider) output surface}.
*/
/* package */ final class FrameEditor {
......@@ -56,7 +55,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param outputWidth The output width in pixels.
* @param outputHeight The output height in pixels.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame.
* @param outputSurface The {@link Surface}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param debugViewProvider Provider for optional debug views to show intermediate output.
......@@ -65,12 +64,14 @@ import java.util.concurrent.atomic.AtomicInteger;
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components.
*/
// TODO(b/214975934): Take a List<GlFrameProcessor> as input and rename FrameEditor to
// FrameProcessorChain.
public static FrameEditor create(
Context context,
int outputWidth,
int outputHeight,
float pixelWidthHeightRatio,
Matrix transformationMatrix,
GlFrameProcessor transformationFrameProcessor,
Surface outputSurface,
boolean enableExperimentalHdrEditing,
Transformer.DebugViewProvider debugViewProvider)
......@@ -85,8 +86,8 @@ import java.util.concurrent.atomic.AtomicInteger;
TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
GlFrameProcessor frameProcessor =
new GlFrameProcessor(context, transformationMatrix, enableExperimentalHdrEditing);
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
......@@ -94,7 +95,9 @@ import java.util.concurrent.atomic.AtomicInteger;
EGLDisplay eglDisplay;
EGLContext eglContext;
EGLSurface eglSurface;
int textureId;
int inputExternalTexId;
int intermediateTexId;
int frameBuffer;
@Nullable EGLSurface debugPreviewEglSurface = null;
try {
eglDisplay = GlUtil.createEglDisplay();
......@@ -117,9 +120,12 @@ import java.util.concurrent.atomic.AtomicInteger;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
textureId = GlUtil.createExternalTexture();
frameProcessor.initialize();
} catch (IOException | GlUtil.GlException e) {
inputExternalTexId = GlUtil.createExternalTexture();
intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight);
frameBuffer = GlUtil.createFboForTexture(intermediateTexId);
externalCopyFrameProcessor.initialize();
transformationFrameProcessor.initialize();
} catch (GlUtil.GlException | IOException e) {
throw TransformationException.createForFrameEditor(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
......@@ -138,8 +144,11 @@ import java.util.concurrent.atomic.AtomicInteger;
eglDisplay,
eglContext,
eglSurface,
textureId,
frameProcessor,
inputExternalTexId,
intermediateTexId,
frameBuffer,
externalCopyFrameProcessor,
transformationFrameProcessor,
outputWidth,
outputHeight,
debugPreviewEglSurface,
......@@ -147,12 +156,25 @@ import java.util.concurrent.atomic.AtomicInteger;
debugPreviewHeight);
}
private final GlFrameProcessor frameProcessor;
// TODO(b/214975934): Write javadoc for fields where the purpose might be unclear to someone less
// familiar with this class and consider grouping some of these fields into new classes to
// reduce the number of constructor parameters.
private final ExternalCopyFrameProcessor externalCopyFrameProcessor;
private final GlFrameProcessor transformationFrameProcessor;
private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final EGLSurface eglSurface;
private final int textureId;
/** Indentifier of the external texture the {@code FrameEditor} reads its input from. */
private final int inputExternalTexId;
/**
* Indentifier of the texture where the output of the {@link ExternalCopyFrameProcessor} is
* written to and the {@link TransformationFrameProcessor} reads its input from.
*/
private final int intermediateTexId;
/** Identifier of a framebuffer object associated with the intermediate texture. */
private final int frameBuffer;
private final AtomicInteger pendingInputFrameCount;
private final AtomicInteger availableInputFrameCount;
private final SurfaceTexture inputSurfaceTexture;
......@@ -169,8 +191,11 @@ import java.util.concurrent.atomic.AtomicInteger;
EGLDisplay eglDisplay,
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
GlFrameProcessor frameProcessor,
int inputExternalTexId,
int intermediateTexId,
int frameBuffer,
ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor,
int outputWidth,
int outputHeight,
@Nullable EGLSurface debugPreviewEglSurface,
......@@ -179,8 +204,11 @@ import java.util.concurrent.atomic.AtomicInteger;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.frameProcessor = frameProcessor;
this.inputExternalTexId = inputExternalTexId;
this.intermediateTexId = intermediateTexId;
this.frameBuffer = frameBuffer;
this.externalCopyFrameProcessor = externalCopyFrameProcessor;
this.transformationFrameProcessor = transformationFrameProcessor;
this.outputWidth = outputWidth;
this.outputHeight = outputHeight;
this.debugPreviewEglSurface = debugPreviewEglSurface;
......@@ -189,7 +217,7 @@ import java.util.concurrent.atomic.AtomicInteger;
pendingInputFrameCount = new AtomicInteger();
availableInputFrameCount = new AtomicInteger();
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture = new SurfaceTexture(inputExternalTexId);
inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> {
checkState(pendingInputFrameCount.getAndDecrement() > 0);
......@@ -235,10 +263,15 @@ import java.util.concurrent.atomic.AtomicInteger;
try {
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
frameProcessor.setTextureTransformMatrix(textureTransformMatrix);
frameProcessor.updateProgramAndDraw(textureId);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
GlUtil.focusFramebuffer(
eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight);
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.updateProgramAndDraw(inputExternalTexId, presentationTimeNs);
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
transformationFrameProcessor.updateProgramAndDraw(intermediateTexId, presentationTimeNs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
......@@ -260,8 +293,10 @@ import java.util.concurrent.atomic.AtomicInteger;
/** Releases all resources. */
public void release() {
frameProcessor.release();
GlUtil.deleteTexture(textureId);
externalCopyFrameProcessor.release();
transformationFrameProcessor.release();
GlUtil.deleteTexture(inputExternalTexId);
GlUtil.deleteTexture(intermediateTexId);
GlUtil.destroyEglContext(eglDisplay, eglContext);
inputSurfaceTexture.release();
inputSurface.release();
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java
......@@ -15,98 +15,13 @@
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.opengl.GLES20;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Manages a GLSL shader program for applying a transformation matrix to a frame. */
/* package */ class GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
/**
* Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful
* for converting to the 4x4 column-major format commonly used in OpenGL.
*/
private static float[] getGlMatrixArray(Matrix matrix) {
float[] matrix3x3Array = new float[9];
matrix.getValues(matrix3x3Array);
float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array);
// Transpose from row-major to column-major representations.
float[] transposedMatrix4x4Array = new float[16];
android.opengl.Matrix.transposeM(
transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0);
return transposedMatrix4x4Array;
}
/**
* Returns a 4x4 matrix array containing the 3x3 matrix array's contents.
*
* <p>The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected
* to be in 3 dimensions. The output will have the third row/column's values be an identity
* matrix's values, so that vertex transformations using this matrix will not affect the z axis.
* <br>
* Input format: [a, b, c, d, e, f, g, h, i] <br>
* Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i]
*/
private static float[] getMatrix4x4Array(float[] matrix3x3Array) {
float[] matrix4x4Array = new float[16];
matrix4x4Array[10] = 1;
for (int inputRow = 0; inputRow < 3; inputRow++) {
for (int inputColumn = 0; inputColumn < 3; inputColumn++) {
int outputRow = (inputRow == 2) ? 3 : inputRow;
int outputColumn = (inputColumn == 2) ? 3 : inputColumn;
matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn];
}
}
return matrix4x4Array;
}
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external.glsl";
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final Context context;
private final Matrix transformationMatrix;
private final boolean enableExperimentalHdrEditing;
private @MonotonicNonNull GlProgram glProgram;
/** Manages a GLSL shader program for processing a frame. */
/* package */ interface GlFrameProcessor {
/**
* Creates a new instance.
*
* @param context A {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
*/
public GlFrameProcessor(
Context context, Matrix transformationMatrix, boolean enableExperimentalHdrEditing) {
this.context = context;
this.transformationMatrix = transformationMatrix;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
// TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension
// calculations out of the VideoTranscodingSamplePipeline into the frame processors.
/**
* Does any initialization necessary such as loading and compiling a GLSL shader programs.
......@@ -114,43 +29,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This method may only be called after creating the OpenGL context and focusing a render
* target.
*/
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
String vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
if (enableExperimentalHdrEditing) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray);
}
/**
* Sets the texture transform matrix for converting an external surface texture's coordinates to
* sampling locations.
*
* @param textureTransformMatrix The external surface texture's {@link
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
void initialize() throws IOException;
/**
* Updates the shader program's vertex attributes and uniforms, binds them, and draws.
......@@ -159,23 +38,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* focussing the correct render target before calling this method.
*
* @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from.
* @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds.
*/
// TODO(b/214975934): Also pass presentationTimeNs.
public void updateProgramAndDraw(int inputTexId) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
void updateProgramAndDraw(int inputTexId, long presentationTimeNs);
/** Releases all resources. */
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
void release();
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessor.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.opengl.GLES20;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Applies a transformation matrix in the vertex shader. */
/* package */ class TransformationFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
/**
* Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful
* for converting to the 4x4 column-major format commonly used in OpenGL.
*/
private static float[] getGlMatrixArray(Matrix matrix) {
float[] matrix3x3Array = new float[9];
matrix.getValues(matrix3x3Array);
float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array);
// Transpose from row-major to column-major representations.
float[] transposedMatrix4x4Array = new float[16];
android.opengl.Matrix.transposeM(
transposedMatrix4x4Array, /* mTransOffset= */ 0, matrix4x4Array, /* mOffset= */ 0);
return transposedMatrix4x4Array;
}
/**
* Returns a 4x4 matrix array containing the 3x3 matrix array's contents.
*
* <p>The 3x3 matrix array is expected to be in 2 dimensions, and the 4x4 matrix array is expected
* to be in 3 dimensions. The output will have the third row/column's values be an identity
* matrix's values, so that vertex transformations using this matrix will not affect the z axis.
* <br>
* Input format: [a, b, c, d, e, f, g, h, i] <br>
* Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i]
*/
private static float[] getMatrix4x4Array(float[] matrix3x3Array) {
float[] matrix4x4Array = new float[16];
matrix4x4Array[10] = 1;
for (int inputRow = 0; inputRow < 3; inputRow++) {
for (int inputColumn = 0; inputColumn < 3; inputColumn++) {
int outputRow = (inputRow == 2) ? 3 : inputRow;
int outputColumn = (inputColumn == 2) ? 3 : inputColumn;
matrix4x4Array[outputRow * 4 + outputColumn] = matrix3x3Array[inputRow * 3 + inputColumn];
}
}
return matrix4x4Array;
}
private final Context context;
private final Matrix transformationMatrix;
private @MonotonicNonNull GlProgram glProgram;
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
*/
public TransformationFrameProcessor(Context context, Matrix transformationMatrix) {
this.context = context;
this.transformationMatrix = transformationMatrix;
}
@Override
public void initialize() throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms
// expected in the code.
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setFloatsUniform("uTransformationMatrix", getGlMatrixArray(transformationMatrix));
}
@Override
public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) {
checkStateNotNull(glProgram);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0);
glProgram.use();
glProgram.bindAttributesAndUniforms();
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
......@@ -166,7 +166,7 @@ import org.checkerframework.dataflow.qual.Pure;
encoderSupportedFormat.width,
encoderSupportedFormat.height,
inputFormat.pixelWidthHeightRatio,
transformationMatrix,
new TransformationFrameProcessor(context, transformationMatrix),
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment