Commit 0615922c by hschlueter Committed by Rohit Singh

Merge MatrixTransformationProcessor and ExternalTextureProcessor.

This saves an intermediate texture copy step for use-cases
where matrix transformations are the first or only effects
in the chain.

PiperOrigin-RevId: 460239403
parent 6e126000
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
attribute vec4 aFramePosition;
uniform mat4 uTexTransform;
varying vec2 vTexSamplingCoord;
void main() {
gl_Position = aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
}
...@@ -13,13 +13,15 @@ ...@@ -13,13 +13,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// ES 2 vertex shader that applies the 4 * 4 transformation matrix // ES 2 vertex shader that applies the 4 * 4 transformation matrices
// uTransformationMatrix. // uTransformationMatrix and the uTexTransformationMatrix.
attribute vec4 aFramePosition; attribute vec4 aFramePosition;
uniform mat4 uTransformationMatrix; uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
varying vec2 vTexSamplingCoord; varying vec2 vTexSamplingCoord;
void main() { void main() {
gl_Position = uTransformationMatrix * aFramePosition; gl_Position = uTransformationMatrix * aFramePosition;
vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5); vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
} }
#version 300 es #version 300 es
// Copyright 2022 The Android Open Source Project // Copyright 2021 The Android Open Source Project
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
...@@ -13,15 +13,15 @@ ...@@ -13,15 +13,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture // ES 3 vertex shader that applies the 4 * 4 transformation matrices
// transformation matrix to convert the texture coordinates to the sampling // uTransformationMatrix and the uTexTransformationMatrix.
// locations.
in vec4 aFramePosition; in vec4 aFramePosition;
uniform mat4 uTexTransform; uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
out vec2 vTexSamplingCoord; out vec2 vTexSamplingCoord;
void main() { void main() {
gl_Position = aFramePosition; gl_Position = uTransformationMatrix * aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0); vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy; vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
} }
...@@ -15,71 +15,13 @@ ...@@ -15,71 +15,13 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; /**
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; * Interface for a {@link GlTextureProcessor} that samples from an external texture.
import android.content.Context;
import android.opengl.GLES20;
import android.util.Size;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
"shaders/vertex_shader_tex_transform_es2.glsl";
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
"shaders/vertex_shader_tex_transform_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final GlProgram glProgram;
/**
* Creates a new instance.
* *
* @param useHdr Whether to process the input as an HDR signal. * <p>Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation
* @throws FrameProcessingException If a problem occurs while reading shader files. * matrix.
*/ */
public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException { /* package */ interface ExternalTextureProcessor extends GlTextureProcessor {
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
}
@Override
public Size configure(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
return new Size(inputWidth, inputHeight);
}
/** /**
* Sets the texture transform matrix for converting an external surface texture's coordinates to * Sets the texture transform matrix for converting an external surface texture's coordinates to
...@@ -88,35 +30,5 @@ import java.io.IOException; ...@@ -88,35 +30,5 @@ import java.io.IOException;
* @param textureTransformMatrix The external surface texture's {@linkplain * @param textureTransformMatrix The external surface texture's {@linkplain
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}. * android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/ */
public void setTextureTransformMatrix(float[] textureTransformMatrix) { void setTextureTransformMatrix(float[] textureTransformMatrix);
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
if (glProgram != null) {
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
}
} }
...@@ -24,6 +24,7 @@ import android.opengl.EGLDisplay; ...@@ -24,6 +24,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Size; import android.util.Size;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceHolder; import android.view.SurfaceHolder;
...@@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link * <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
* GlTextureProcessor} instances used by {@link FrameProcessor}. * GlTextureProcessor} instances used by {@link FrameProcessor}.
*/ */
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor { /* package */ final class FinalMatrixTransformationProcessorWrapper
implements GlTextureProcessor, ExternalTextureProcessor {
private static final String TAG = "FinalProcessorWrapper"; private static final String TAG = "FinalProcessorWrapper";
...@@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final long streamOffsetUs; private final long streamOffsetUs;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener; private final FrameProcessor.Listener frameProcessorListener;
private final boolean sampleFromExternalTexture;
private final boolean useHdr; private final boolean useHdr;
private final float[] textureTransformMatrix;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
...@@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener, FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
boolean useHdr) { boolean useHdr) {
this.context = context; this.context = context;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
...@@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener; this.frameProcessorListener = frameProcessorListener;
this.sampleFromExternalTexture = sampleFromExternalTexture;
this.useHdr = useHdr; this.useHdr = useHdr;
textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
} }
/** /**
...@@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
MatrixTransformationProcessor matrixTransformationProcessor = MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build()); new MatrixTransformationProcessor(
context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight); Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
checkState(outputSize.getWidth() == outputSurfaceInfo.width); checkState(outputSize.getWidth() == outputSurfaceInfo.width);
checkState(outputSize.getHeight() == outputSurfaceInfo.height); checkState(outputSize.getHeight() == outputSurfaceInfo.height);
...@@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
System.arraycopy(
/* src= */ textureTransformMatrix,
/* srcPos= */ 0,
/* dest= */ this.textureTransformMatrix,
/* destPost= */ 0,
/* length= */ textureTransformMatrix.length);
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
}
}
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
this.outputSurfaceInfo = outputSurfaceInfo; this.outputSurfaceInfo = outputSurfaceInfo;
......
...@@ -36,19 +36,35 @@ import java.util.Arrays; ...@@ -36,19 +36,35 @@ import java.util.Arrays;
* matrices are clipped to the NDC range. * matrices are clipped to the NDC range.
* *
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0). * <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/ */
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298 @SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor { /* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
implements ExternalTextureProcessor {
private static final String VERTEX_SHADER_TRANSFORMATION_PATH = private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation_es2.glsl"; "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
private static final ImmutableList<float[]> NDC_SQUARE = private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of( ImmutableList.of(
new float[] {-1, -1, 0, 1}, new float[] {-1, -1, 0, 1},
new float[] {-1, 1, 0, 1}, new float[] {-1, 1, 0, 1},
new float[] {1, 1, 0, 1}, new float[] {1, 1, 0, 1},
new float[] {1, -1, 0, 1}); new float[] {1, -1, 0, 1});
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
/** The {@link MatrixTransformation MatrixTransformations} to apply. */ /** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations; private final ImmutableList<GlMatrixTransformation> matrixTransformations;
...@@ -87,7 +103,11 @@ import java.util.Arrays; ...@@ -87,7 +103,11 @@ import java.util.Arrays;
*/ */
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation)); this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
} }
/** /**
...@@ -100,7 +120,11 @@ import java.util.Arrays; ...@@ -100,7 +120,11 @@ import java.util.Arrays;
*/ */
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation)); this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
} }
/** /**
...@@ -109,10 +133,17 @@ import java.util.Arrays; ...@@ -109,10 +133,17 @@ import java.util.Arrays;
* @param context The {@link Context}. * @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. * apply to each frame in order.
* @param sampleFromExternalTexture Whether the input will be provided using an external texture.
* If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to
* provide the transformation matrix associated with the external texture.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor( public MatrixTransformationProcessor(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations) Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean sampleFromExternalTexture,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException { throws FrameProcessingException {
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
...@@ -121,11 +152,41 @@ import java.util.Arrays; ...@@ -121,11 +152,41 @@ import java.util.Arrays;
tempResultMatrix = new float[16]; tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE; visiblePolygon = NDC_SQUARE;
String vertexShaderFilePath;
String fragmentShaderFilePath;
if (sampleFromExternalTexture) {
vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
}
try { try {
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) { } catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e); throw new FrameProcessingException(e);
} }
if (enableExperimentalHdrEditing && sampleFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
}
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
glProgram.setFloatsUniform("uTexTransformationMatrix", textureTransformMatrix);
} }
@Override @Override
......
...@@ -73,16 +73,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso ...@@ -73,16 +73,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
this.listener = listener; this.listener = listener;
} }
/**
* Returns whether the {@code SingleFrameGlTextureProcessor} can accept an input frame.
*
* <p>If this method returns {@code true}, the next call to {@link #maybeQueueInputFrame(
* TextureInfo, long)} will also return {@code true}.
*/
public boolean acceptsInputFrame() {
return !outputTextureInUse;
}
@Override @Override
public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (outputTextureInUse) { if (outputTextureInUse) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment