Skip to content

SDK / exoplayer

Go to a project
Commit 0615922c by hschlueter Committed by Rohit Singh
Options

Merge MatrixTransformationProcessor and ExternalTextureProcessor. 

This saves an intermediate texture copy step for use-cases
where matrix transformations are the first or only effects
in the chain.

PiperOrigin-RevId: 460239403
parent 6e126000
Expand all
Inline Side-by-side
Showing with 114 additions and 151 deletions
library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl deleted 100644 → 0
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
attribute vec4 aFramePosition;
uniform mat4 uTexTransform;
varying vec2 vTexSamplingCoord;
void main() {
gl_Position = aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
}
library/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl
......@@ -13,13 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that applies the 4 * 4 transformation matrix
// uTransformationMatrix.
// ES 2 vertex shader that applies the 4 * 4 transformation matrices
// uTransformationMatrix and the uTexTransformationMatrix.
attribute vec4 aFramePosition;
uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
varying vec2 vTexSamplingCoord;
void main() {
gl_Position = uTransformationMatrix * aFramePosition;
vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5);
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
}
library/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es3.glsl → library/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl
#version 300 es
// Copyright 2022 The Android Open Source Project
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -13,15 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture
// transformation matrix to convert the texture coordinates to the sampling
// locations.
// ES 3 vertex shader that applies the 4 * 4 transformation matrices
// uTransformationMatrix and the uTexTransformationMatrix.
in vec4 aFramePosition;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
uniform mat4 uTexTransformationMatrix;
out vec2 vTexSamplingCoord;
void main() {
gl_Position = aFramePosition;
gl_Position = uTransformationMatrix * aFramePosition;
vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
vTexSamplingCoord = (uTexTransform * texturePosition).xy;
vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java
......@@ -15,71 +15,13 @@
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
import android.util.Size;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
"shaders/vertex_shader_tex_transform_es2.glsl";
private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
"shaders/vertex_shader_tex_transform_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
private final GlProgram glProgram;
/**
* Creates a new instance.
*
* @param useHdr Whether to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
}
@Override
public Size configure(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
return new Size(inputWidth, inputHeight);
}
/**
* Interface for a {@link GlTextureProcessor} that samples from an external texture.
*
* <p>Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation
* matrix.
*/
/* package */ interface ExternalTextureProcessor extends GlTextureProcessor {
/**
* Sets the texture transform matrix for converting an external surface texture's coordinates to
......@@ -88,35 +30,5 @@ import java.io.IOException;
* @param textureTransformMatrix The external surface texture's {@linkplain
* android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
*/
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
checkStateNotNull(glProgram);
glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
if (glProgram != null) {
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
}
void setTextureTransformMatrix(float[] textureTransformMatrix);
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
......@@ -24,6 +24,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceHolder;
......@@ -50,7 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
* GlTextureProcessor} instances used by {@link FrameProcessor}.
*/
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
/* package */ final class FinalMatrixTransformationProcessorWrapper
implements GlTextureProcessor, ExternalTextureProcessor {
private static final String TAG = "FinalProcessorWrapper";
......@@ -61,7 +63,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final long streamOffsetUs;
private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener;
private final boolean sampleFromExternalTexture;
private final boolean useHdr;
private final float[] textureTransformMatrix;
private int inputWidth;
private int inputHeight;
......@@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
boolean useHdr) {
this.context = context;
this.matrixTransformations = matrixTransformations;
......@@ -94,7 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener;
this.sampleFromExternalTexture = sampleFromExternalTexture;
this.useHdr = useHdr;
textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
}
/**
......@@ -239,7 +248,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
new MatrixTransformationProcessor(
context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
checkState(outputSize.getWidth() == outputSurfaceInfo.width);
checkState(outputSize.getHeight() == outputSurfaceInfo.height);
......@@ -265,6 +276,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
System.arraycopy(
/* src= */ textureTransformMatrix,
/* srcPos= */ 0,
/* dest= */ this.textureTransformMatrix,
/* destPost= */ 0,
/* length= */ textureTransformMatrix.length);
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
}
}
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
this.outputSurfaceInfo = outputSurfaceInfo;
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java
......@@ -36,19 +36,35 @@ import java.util.Arrays;
* matrices are clipped to the NDC range.
*
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor {
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
implements ExternalTextureProcessor {
private static final String VERTEX_SHADER_TRANSFORMATION_PATH =
"shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of(
new float[] {-1, -1, 0, 1},
new float[] {-1, 1, 0, 1},
new float[] {1, 1, 0, 1},
new float[] {1, -1, 0, 1});
// Color transform coefficients from
// https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
1.168f, 1.168f, 1.168f,
0.0f, -0.188f, 2.148f,
1.683f, -0.652f, 0.0f,
};
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
......@@ -87,7 +103,11 @@ import java.util.Arrays;
*/
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation));
this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
}
/**
......@@ -100,7 +120,11 @@ import java.util.Arrays;
*/
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException {
this(context, ImmutableList.of(matrixTransformation));
this(
context,
ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false,
/* enableExperimentalHdrEditing= */ false);
}
/**
......@@ -109,10 +133,17 @@ import java.util.Arrays;
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param sampleFromExternalTexture Whether the input will be provided using an external texture.
* If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to
* provide the transformation matrix associated with the external texture.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public MatrixTransformationProcessor(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations)
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean sampleFromExternalTexture,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
this.matrixTransformations = matrixTransformations;
......@@ -121,11 +152,41 @@ import java.util.Arrays;
tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE;
String vertexShaderFilePath;
String fragmentShaderFilePath;
if (sampleFromExternalTexture) {
vertexShaderFilePath =
enableExperimentalHdrEditing
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath =
enableExperimentalHdrEditing
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
}
try {
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
if (enableExperimentalHdrEditing && sampleFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
}
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
}
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
glProgram.setFloatsUniform("uTexTransformationMatrix", textureTransformMatrix);
}
@Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java
......@@ -73,16 +73,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
this.listener = listener;
}
/**
* Returns whether the {@code SingleFrameGlTextureProcessor} can accept an input frame.
*
* <p>If this method returns {@code true}, the next call to {@link #maybeQueueInputFrame(
* TextureInfo, long)} will also return {@code true}.
*/
public boolean acceptsInputFrame() {
return !outputTextureInUse;
}
@Override
public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (outputTextureInUse) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment