Skip to content

SDK / exoplayer

Go to a project
Commit 4c8d22fe by huangdarwin Committed by microkatz
Options

HDR: Use factory for MatrixTransformationProcessor. 

Separate MatrixTransformationProcessor constructors by color input and output.

PiperOrigin-RevId: 471034768
(cherry picked from commit a8e814ab)
parent fc8edfcd
Inline Side-by-side
Showing with 167 additions and 130 deletions
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java
......@@ -28,6 +28,7 @@ import android.opengl.EGLSurface;
import androidx.media3.common.FrameProcessingException;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.util.GlUtil;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
......@@ -94,11 +95,10 @@ public final class MatrixTransformationProcessorPixelTest {
public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits";
Matrix identityMatrix = new Matrix();
MatrixTransformation noEditsTransformation = (long presentationTimeUs) -> identityMatrix;
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> identityMatrix);
MatrixTransformationProcessor.create(
context, ImmutableList.of(noEditsTransformation), /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
......@@ -120,11 +120,11 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_translateRight";
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
MatrixTransformation translateRightTransformation =
(long presentationTimeUs) -> translateRightMatrix;
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix);
MatrixTransformationProcessor.create(
context, ImmutableList.of(translateRightTransformation), /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
......@@ -146,11 +146,10 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_scaleNarrow";
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(.5f, 1.2f);
MatrixTransformation scaleNarrowTransformation = (long presentationTimeUs) -> scaleNarrowMatrix;
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix);
MatrixTransformationProcessor.create(
context, ImmutableList.of(scaleNarrowTransformation), /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
......@@ -172,11 +171,10 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_rotate90";
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
MatrixTransformation rotate90Transformation = (long presentationTimeUs) -> rotate90Matrix;
matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix);
MatrixTransformationProcessor.create(
context, ImmutableList.of(rotate90Transformation), /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
......@@ -368,13 +368,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Presentation.createForWidthAndHeight(
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(
context,
matrixTransformationListBuilder.build(),
sampleFromExternalTexture,
colorInfo,
/* outputElectricalColors= */ true);
MatrixTransformationProcessor matrixTransformationProcessor;
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
matrixTransformationListBuilder.build();
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotfThenOetf(
context, expandedMatrixTransformations, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.createApplyingOetf(
context, expandedMatrixTransformations, colorInfo);
}
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
......@@ -204,13 +204,17 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
textureProcessorListBuilder.add(
new MatrixTransformationProcessor(
context,
matrixTransformations,
sampleFromExternalTexture,
colorInfo,
/* outputElectricalColors= */ false));
MatrixTransformationProcessor matrixTransformationProcessor;
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, matrixTransformations, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context, matrixTransformations, ColorInfo.isTransferHdr(colorInfo));
}
textureProcessorListBuilder.add(matrixTransformationProcessor);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
......@@ -235,12 +239,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
// TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got
// merged with RgbMatrixProcessor.
textureProcessorListBuilder.add(
new MatrixTransformationProcessor(
context,
ImmutableList.of(),
sampleFromExternalTexture,
colorInfo,
/* outputElectricalColors= */ false));
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, /* matrixTransformations= */ ImmutableList.of(), colorInfo));
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java
......@@ -19,6 +19,7 @@ import android.content.Context;
import android.opengl.Matrix;
import android.util.Pair;
import androidx.media3.common.FrameProcessingException;
import com.google.common.collect.ImmutableList;
/**
* Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame.
......@@ -52,6 +53,7 @@ public interface GlMatrixTransformation extends GlEffect {
@Override
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return new MatrixTransformationProcessor(context, useHdr, /* matrixTransformation= */ this);
return MatrixTransformationProcessor.create(
context, /* matrixTransformations= */ ImmutableList.of(this), useHdr);
}
}
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
......@@ -110,97 +110,152 @@ import java.util.Arrays;
/**
* Creates a new instance.
*
* <p>Input and output are both intermediate optical colors, which are linear RGB BT.2020 if
* {@code useHdr} is {@code true} and gamma RGB BT.709 if not.
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param useHdr Whether input and output colors are HDR.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public MatrixTransformationProcessor(
Context context, boolean useHdr, MatrixTransformation matrixTransformation)
public static MatrixTransformationProcessor create(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations, boolean useHdr)
throws FrameProcessingException {
this(
createGlProgram(
context,
/* inputElectricalColorsFromExternalTexture= */ false,
useHdr,
/* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
GlProgram glProgram =
createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_COPY_PATH);
// No transfer functions needed, because input and output are both optical colors.
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
}
/**
* Creates a new instance.
*
* <p>Input will be sampled from an external texture. The caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the
* external texture.
*
* <p>Applies the {@code electricalColorInfo} EOTF to convert from electrical color input, to
* intermediate optical {@link GlTextureProcessor} color output, before {@code
* matrixTransformations} are applied.
*
* <p>Intermediate optical colors are linear RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 if not.
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public MatrixTransformationProcessor(
Context context, boolean useHdr, GlMatrixTransformation matrixTransformation)
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
this(
createGlProgram(
context,
/* inputElectricalColorsFromExternalTexture= */ false,
useHdr,
/* outputElectricalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
electricalColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
}
/**
* Creates a new instance.
*
* <p>Able to convert nonlinear electrical {@link ColorInfo} inputs and outputs to and from the
* intermediate optical {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma
* RGB BT.709 for SDR.
* <p>Applies the {@code electricalColorInfo} OETF to convert from intermediate optical {@link
* GlTextureProcessor} color input, to electrical color output, after {@code
* matrixTransformations} are applied.
*
* <p>Intermediate optical colors are linear RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 if not.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param inputElectricalColorsFromExternalTexture Whether electrical color input will be provided
* using an external texture. If {@code true}, the caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with
* the external texture.
* @param electricalColorInfo The electrical {@link ColorInfo}, only used to transform between
* color spaces and transfers, when {@code inputElectricalColorsFromExternalTexture} or {@code
* outputElectricalColors} are {@code true}. If it is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, intermediate {@link GlTextureProcessor} colors
* will be in linear RGB BT.2020. Otherwise, these colors will be in gamma RGB BT.709.
* @param outputElectricalColors If {@code true}, outputs {@code electricalColorInfo}. If {@code
* false}, outputs intermediate colors of linear RGB BT.2020 if {@code electricalColorInfo} is
* {@linkplain ColorInfo#isTransferHdr(ColorInfo) HDR}, and gamma RGB BT.709 otherwise.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public MatrixTransformationProcessor(
public static MatrixTransformationProcessor createApplyingOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean inputElectricalColorsFromExternalTexture,
ColorInfo electricalColorInfo,
boolean outputElectricalColors)
ColorInfo electricalColorInfo)
throws FrameProcessingException {
this(
createGlProgram(
context,
inputElectricalColorsFromExternalTexture,
ColorInfo.isTransferHdr(electricalColorInfo),
outputElectricalColors),
matrixTransformations,
ColorInfo.isTransferHdr(electricalColorInfo));
if (!ColorInfo.isTransferHdr(electricalColorInfo)) {
return;
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) {
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
}
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
if (inputElectricalColorsFromExternalTexture) {
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
}
/**
* Creates a new instance.
*
* <p>Input will be sampled from an external texture. The caller should use {@link
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the
* external texture.
*
* <p>Applies the OETF, {@code matrixTransformations}, then the EOTF, to convert from and to input
* and output electrical colors.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotfThenOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
......@@ -212,14 +267,11 @@ import java.util.Arrays;
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
// TODO(b/241902517): Implement gamma transfer functions.
// If electrical colors are both input and output, no EOTF is needed.
glProgram.setIntUniform(
"uEotfColorTransfer", outputElectricalColors ? Format.NO_VALUE : colorTransfer);
} else if (outputElectricalColors) {
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
// No transfer functions needed, because the EOTF and OETF cancel out.
glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
}
/**
......@@ -241,36 +293,15 @@ import java.util.Arrays;
transformationMatrixCache = new float[matrixTransformations.size()][16];
compositeTransformationMatrix = new float[16];
tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
tempResultMatrix = new float[16];
visiblePolygon = NDC_SQUARE;
}
private static GlProgram createGlProgram(
Context context,
boolean inputElectricalColorsFromExternalTexture,
boolean useHdr,
boolean outputElectricalColors)
Context context, String vertexShaderFilePath, String fragmentShaderFilePath)
throws FrameProcessingException {
String vertexShaderFilePath;
String fragmentShaderFilePath;
if (inputElectricalColorsFromExternalTexture) {
if (useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
}
} else if (outputElectricalColors && useHdr) {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_OETF_ES3_PATH;
} else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
}
GlProgram glProgram;
try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment