Skip to content

SDK / exoplayer

Go to a project
Commit 672405af by leonwind Committed by Marc Baechinger
Options

Merge RgbProcessor and MatrixTransformation. 

PiperOrigin-RevId: 472325145
parent 3a2e0d37
Inline Side-by-side
Showing with 207 additions and 298 deletions
libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java
......@@ -28,7 +28,6 @@ import android.opengl.EGLSurface;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
......@@ -97,8 +96,7 @@ public final class MatrixTransformationProcessorPixelTest {
Matrix identityMatrix = new Matrix();
MatrixTransformation noEditsTransformation = (long presentationTimeUs) -> identityMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(noEditsTransformation), /* useHdr= */ false);
noEditsTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
......@@ -123,8 +121,7 @@ public final class MatrixTransformationProcessorPixelTest {
MatrixTransformation translateRightTransformation =
(long presentationTimeUs) -> translateRightMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(translateRightTransformation), /* useHdr= */ false);
translateRightTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
......@@ -148,8 +145,7 @@ public final class MatrixTransformationProcessorPixelTest {
scaleNarrowMatrix.postScale(.5f, 1.2f);
MatrixTransformation scaleNarrowTransformation = (long presentationTimeUs) -> scaleNarrowMatrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(scaleNarrowTransformation), /* useHdr= */ false);
scaleNarrowTransformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
......@@ -173,8 +169,7 @@ public final class MatrixTransformationProcessorPixelTest {
rotate90Matrix.postRotate(/* degrees= */ 90);
MatrixTransformation rotate90Transformation = (long presentationTimeUs) -> rotate90Matrix;
matrixTransformationFrameProcessor =
MatrixTransformationProcessor.create(
context, ImmutableList.of(rotate90Transformation), /* useHdr= */ false);
rotate90Transformation.toGlTextureProcessor(context, /* useHdr= */ false);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
......
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
......@@ -62,7 +62,7 @@ public final class RgbAdjustmentPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId;
private int outputTexId;
......@@ -94,8 +94,8 @@ public final class RgbAdjustmentPixelTest {
@After
public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbMatrixProcessor != null) {
rgbMatrixProcessor.release();
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
......@@ -104,11 +104,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_identityMatrix_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_identityMatrix";
RgbMatrix identityMatrix = new RgbAdjustment.Builder().build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, identityMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
identityMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -126,13 +128,15 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_removeColors";
RgbMatrix removeColorMatrix =
new RgbAdjustment.Builder().setRedScale(0).setGreenScale(0).setBlueScale(0).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, removeColorMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
removeColorMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -149,11 +153,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_redOnlyFilter_removeBlueAndGreenValues() throws Exception {
String testId = "drawFrame_redOnlyFilter";
RgbMatrix redOnlyMatrix = new RgbAdjustment.Builder().setBlueScale(0).setGreenScale(0).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, redOnlyMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
redOnlyMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -170,11 +176,13 @@ public final class RgbAdjustmentPixelTest {
public void drawFrame_increaseRedChannel_producesBrighterAndRedderFrame() throws Exception {
String testId = "drawFrame_increaseRedChannel";
RgbMatrix increaseRedMatrix = new RgbAdjustment.Builder().setRedScale(5).build();
rgbMatrixProcessor = new RgbMatrixProcessor(context, increaseRedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
increaseRedMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -192,12 +200,13 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_increaseBrightness";
RgbMatrix increaseBrightnessMatrix =
new RgbAdjustment.Builder().setRedScale(5).setGreenScale(5).setBlueScale(5).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(context, increaseBrightnessMatrix, /* useHdr = */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
increaseBrightnessMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -216,15 +225,19 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix noRed = new RgbAdjustment.Builder().setRedScale(0).build();
RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(
context, ImmutableList.of(noRed, noGreen, noBlue), /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(noRed, noGreen, noBlue),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap =
BitmapTestUtil.createArgb8888BitmapWithSolidColor(
outputSize.first, outputSize.second, Color.BLACK);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -242,12 +255,17 @@ public final class RgbAdjustmentPixelTest {
String testId = "drawFrame_removeBlueAndGreenValuesInAChain";
RgbMatrix noGreen = new RgbAdjustment.Builder().setGreenScale(0).build();
RgbMatrix noBlue = new RgbAdjustment.Builder().setBlueScale(0).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(context, ImmutableList.of(noGreen, noBlue), /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(noGreen, noBlue),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ONLY_RED_CHANNEL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -267,15 +285,17 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix scaleRedMatrix = new RgbAdjustment.Builder().setRedScale(redScale).build();
RgbMatrix scaleRedByInverseMatrix =
new RgbAdjustment.Builder().setRedScale(1 / redScale).build();
rgbMatrixProcessor =
new RgbMatrixProcessor(
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context,
ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix),
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(scaleRedMatrix, scaleRedByInverseMatrix),
/* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java
......@@ -58,7 +58,7 @@ public final class RgbFilterPixelTest {
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull SingleFrameGlTextureProcessor rgbMatrixProcessor;
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
private int inputTexId;
private int outputTexId;
......@@ -90,8 +90,8 @@ public final class RgbFilterPixelTest {
@After
public void release() throws GlUtil.GlException, FrameProcessingException {
if (rgbMatrixProcessor != null) {
rgbMatrixProcessor.release();
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
......@@ -100,11 +100,13 @@ public final class RgbFilterPixelTest {
public void drawFrame_grayscale_producesGrayscaleImage() throws Exception {
String testId = "drawFrame_grayscale";
RgbMatrix grayscaleMatrix = RgbFilter.createGrayscaleFilter();
rgbMatrixProcessor = new RgbMatrixProcessor(context, grayscaleMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
grayscaleMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(GRAYSCALE_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......@@ -121,11 +123,13 @@ public final class RgbFilterPixelTest {
public void drawFrame_inverted_producesInvertedFrame() throws Exception {
String testId = "drawFrame_inverted";
RgbMatrix invertedMatrix = RgbFilter.createInvertedFilter();
rgbMatrixProcessor = new RgbMatrixProcessor(context, invertedMatrix, /* useHdr= */ false);
Pair<Integer, Integer> outputSize = rgbMatrixProcessor.configure(inputWidth, inputHeight);
matrixTransformationProcessor =
invertedMatrix.toGlTextureProcessor(context, /* useHdr= */ false);
Pair<Integer, Integer> outputSize =
matrixTransformationProcessor.configure(inputWidth, inputHeight);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(INVERT_PNG_ASSET_PATH);
rgbMatrixProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
matrixTransformationProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputSize.first, outputSize.second);
......
libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl deleted 100644 → 0
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler,
// copying from this texture to the current output.
precision mediump float;
uniform sampler2D uTexSampler;
varying vec2 vTexSamplingCoord;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord);
}
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl
......@@ -15,7 +15,8 @@
// ES 3 fragment shader that:
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// uTexSampler,
// uTexSampler, and applies a 4x4 RGB color matrix to change the pixel
// colors,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// and
// 3. copies this converted texture color to the current output.
......@@ -28,6 +29,7 @@ out vec4 outColor;
// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
uniform int uOetfColorTransfer;
uniform mat3 uColorTransform;
uniform mat4 uRgbMatrix;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
......@@ -83,5 +85,6 @@ highp vec3 getElectricalColor(highp vec3 linearColor) {
void main() {
vec4 inputColor = texture(uTexSampler, vTexSamplingCoord);
outColor = vec4(getElectricalColor(inputColor.rgb), inputColor.a);
vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1);
outColor = vec4(getElectricalColor(transformedColors.rgb), inputColor.a);
}
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
......@@ -19,11 +19,11 @@
precision mediump float;
uniform sampler2D uTexSampler;
uniform mat4 uColorMatrix;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uColorMatrix * vec4(inputColor.rgb, 1);
gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
}
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl
......@@ -14,12 +14,17 @@
// limitations under the License.
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output.
// copying from this texture to the current output while applying a 4x4 RGB
// color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
void main() {
gl_FragColor = texture2D(uTexSampler, vTexSamplingCoord);
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
}
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl
......@@ -23,13 +23,15 @@
// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical
// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// intermediate shaders by applying the HLG or PQ EOTF.
// 4. Copies this converted texture color to the current output, with alpha = 1.
// 4. Copies this converted texture color to the current output, with alpha = 1,
// while applying a 4x4 RGB color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require
precision mediump float;
uniform __samplerExternal2DY2YEXT uTexSampler;
uniform mat3 uYuvToRgbColorTransform;
uniform mat4 uRgbMatrix;
// C.java#ColorTransfer value.
uniform int uEotfColorTransfer;
in vec2 vTexSamplingCoord;
......@@ -101,5 +103,5 @@ vec3 yuvToRgb(vec3 yuv) {
void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
vec3 rgb = yuvToRgb(srcYuv);
outColor = vec4(getOpticalColor(rgb), 1.0);
outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0);
}
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
......@@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Context context;
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
private final ImmutableList<RgbMatrix> rgbMatrices;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
......@@ -100,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLDisplay eglDisplay,
EGLContext eglContext,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
......@@ -107,6 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
boolean releaseFramesAutomatically) {
this.context = context;
this.matrixTransformations = matrixTransformations;
this.rgbMatrices = rgbMatrices;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
......@@ -374,11 +377,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotfThenOetf(
context, expandedMatrixTransformations, colorInfo);
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.createApplyingOetf(
context, expandedMatrixTransformations, colorInfo);
context, expandedMatrixTransformations, rgbMatrices, colorInfo);
}
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
......
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
......@@ -184,8 +184,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbaMatrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
for (int i = 0; i < effects.size(); i++) {
Effect effect = effects.get(i);
......@@ -200,62 +199,39 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
continue;
}
if (glEffect instanceof RgbMatrix) {
rgbaMatrixTransformationListBuilder.add((RgbMatrix) glEffect);
rgbMatrixListBuilder.add((RgbMatrix) glEffect);
continue;
}
ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) {
MatrixTransformationProcessor matrixTransformationProcessor;
if (sampleFromExternalTexture) {
matrixTransformationProcessor =
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, matrixTransformations, colorInfo);
context, matrixTransformations, rgbMatrices, colorInfo);
} else {
matrixTransformationProcessor =
MatrixTransformationProcessor.create(
context, matrixTransformations, ColorInfo.isTransferHdr(colorInfo));
context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo));
}
textureProcessorListBuilder.add(matrixTransformationProcessor);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
}
ImmutableList<RgbMatrix> rgbaMatrixTransformations =
rgbaMatrixTransformationListBuilder.build();
if (!rgbaMatrixTransformations.isEmpty()) {
textureProcessorListBuilder.add(
new RgbMatrixProcessor(
context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
rgbaMatrixTransformationListBuilder = new ImmutableList.Builder<>();
}
textureProcessorListBuilder.add(
glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo)));
}
ImmutableList<RgbMatrix> rgbaMatrixTransformations =
rgbaMatrixTransformationListBuilder.build();
if (!rgbaMatrixTransformations.isEmpty()) {
// Add a MatrixTransformationProcessor if none yet exists for sampling from an external
// texture.
if (sampleFromExternalTexture) {
// TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got
// merged with RgbMatrixProcessor.
textureProcessorListBuilder.add(
MatrixTransformationProcessor.createWithExternalSamplerApplyingEotf(
context, /* matrixTransformations= */ ImmutableList.of(), colorInfo));
sampleFromExternalTexture = false;
}
textureProcessorListBuilder.add(
new RgbMatrixProcessor(
context, rgbaMatrixTransformations, ColorInfo.isTransferHdr(colorInfo)));
}
textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper(
context,
eglDisplay,
eglContext,
matrixTransformationListBuilder.build(),
rgbMatrixListBuilder.build(),
listener,
debugViewProvider,
sampleFromExternalTexture,
......
libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java
......@@ -56,6 +56,9 @@ public interface GlMatrixTransformation extends GlEffect {
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return MatrixTransformationProcessor.create(
context, /* matrixTransformations= */ ImmutableList.of(this), useHdr);
context,
/* matrixTransformations= */ ImmutableList.of(this),
/* rgbMatrices= */ ImmutableList.of(),
useHdr);
}
}
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
......@@ -34,17 +34,22 @@ import java.io.IOException;
import java.util.Arrays;
/**
* Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into
* an output frame based on their locations after applying the sequence of transformation matrices.
* Applies a sequence of {@link MatrixTransformation MatrixTransformations} in the vertex shader and
* a sequence of {@link RgbMatrix RgbMatrices} in the fragment shader. Copies input pixels into an
* output frame based on their locations after applying the sequence of transformation matrices.
*
* <p>Operations are done on normalized device coordinates (-1 to 1 on x, y, and z axes).
* Transformed vertices that are moved outside of this range after any of the transformation
* matrices are clipped to the NDC range.
* <p>{@link MatrixTransformation} operations are done on normalized device coordinates (-1 to 1 on
* x, y, and z axes). Transformed vertices that are moved outside of this range after any of the
* transformation matrices are clipped to the NDC range.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space (e.g. BT.709 for SDR). Intermediate results are not clamped.
*
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/
// TODO(b/239757183): Rename Matrix to a more generic name.
@UnstableApi
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor
......@@ -54,13 +59,14 @@ import java.util.Arrays;
"shaders/vertex_shader_transformation_es2.glsl";
private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
"shaders/vertex_shader_transformation_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_PATH =
"shaders/fragment_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
"shaders/fragment_shader_oetf_es3.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
"shaders/fragment_shader_copy_external_es2.glsl";
private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_copy_external_yuv_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH =
"shaders/fragment_shader_transformation_external_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_transformation_external_yuv_es3.glsl";
private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of(
new float[] {-1, -1, 0, 1},
......@@ -84,6 +90,10 @@ import java.util.Arrays;
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
/** The {@link RgbMatrix RgbMatrices} to apply. */
private final ImmutableList<RgbMatrix> rgbMatrices;
/** Whether the frame is in HDR or not. */
private final boolean useHdr;
/**
* The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations}
* for the most recent frame.
......@@ -117,19 +127,25 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether input and output colors are HDR.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixTransformationProcessor create(
Context context, ImmutableList<GlMatrixTransformation> matrixTransformations, boolean useHdr)
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
boolean useHdr)
throws FrameProcessingException {
GlProgram glProgram =
createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_COPY_PATH);
createGlProgram(
context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
// No transfer functions needed, because input and output are both optical colors.
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
......@@ -148,7 +164,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
......@@ -156,13 +174,16 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
......@@ -184,7 +205,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
......@@ -199,7 +220,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
......@@ -207,13 +230,14 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createApplyingOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
......@@ -224,7 +248,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
......@@ -239,7 +263,9 @@ import java.util.Arrays;
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
......@@ -247,13 +273,16 @@ import java.util.Arrays;
public static MatrixTransformationProcessor createWithExternalSamplerApplyingEotfThenOetf(
Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
ColorInfo electricalColorInfo)
throws FrameProcessingException {
boolean useHdr = ColorInfo.isTransferHdr(electricalColorInfo);
String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
......@@ -273,7 +302,7 @@ import java.util.Arrays;
glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE);
}
return new MatrixTransformationProcessor(glProgram, matrixTransformations, useHdr);
return new MatrixTransformationProcessor(glProgram, matrixTransformations, rgbMatrices, useHdr);
}
/**
......@@ -281,17 +310,22 @@ import java.util.Arrays;
*
* @param glProgram The {@link GlProgram}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
*/
private MatrixTransformationProcessor(
GlProgram glProgram,
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
boolean useHdr) {
super(useHdr);
this.glProgram = glProgram;
this.matrixTransformations = matrixTransformations;
this.rgbMatrices = rgbMatrices;
this.useHdr = useHdr;
transformationMatrixCache = new float[matrixTransformations.size()][16];
compositeTransformationMatrix = new float[16];
......@@ -333,11 +367,13 @@ import java.util.Arrays;
if (visiblePolygon.size() < 3) {
return; // Need at least three visible vertices for a triangle.
}
float[] compositeRgbMatrix =
createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix);
glProgram.setFloatsUniform("uRgbMatrix", compositeRgbMatrix);
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.createVertexBuffer(visiblePolygon),
......@@ -425,4 +461,31 @@ import java.util.Arrays;
}
return matrixChanged;
}
// TODO(b/239757183): Add caching for RgbMatrix and refactor RgbMatrix and MatrixTransformation
// composing.
private static float[] createCompositeRgbaMatrixArray(
ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr, long presentationTimeUs) {
float[] tempResultMatrix = new float[16];
float[] compositeRgbaMatrix = new float[16];
Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0);
for (int i = 0; i < rgbMatrices.size(); i++) {
Matrix.multiplyMM(
/* result= */ tempResultMatrix,
/* resultOffset= */ 0,
/* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr),
/* lhsOffset= */ 0,
/* rhs= */ compositeRgbaMatrix,
/* rhsOffset= */ 0);
System.arraycopy(
/* src= */ tempResultMatrix,
/* srcPos= */ 0,
/* dest= */ compositeRgbaMatrix,
/* destPost= */ 0,
/* length= */ tempResultMatrix.length);
}
return compositeRgbaMatrix;
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java
......@@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Provides common color filters. */
......@@ -91,9 +92,13 @@ public class RgbFilter implements RgbMatrix {
}
@Override
public RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
public MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
checkForConsistentHdrSetting(useHdr);
return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
return MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(this),
useHdr);
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
......@@ -19,6 +19,7 @@ package androidx.media3.effect;
import android.content.Context;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
/**
* Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
......@@ -39,8 +40,12 @@ public interface RgbMatrix extends GlEffect {
float[] getMatrix(long presentationTimeUs, boolean useHdr);
@Override
default RgbMatrixProcessor toGlTextureProcessor(Context context, boolean useHdr)
default MatrixTransformationProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException {
return new RgbMatrixProcessor(context, /* rgbMatrix= */ this, useHdr);
return MatrixTransformationProcessor.create(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(this),
useHdr);
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java deleted 100644 → 0
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import android.content.Context;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Pair;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
/**
* Applies a sequence of {@link RgbMatrix} to each frame.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space. Intermediate results are not clamped.
*/
/* package */ final class RgbMatrixProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH =
"shaders/fragment_shader_transformation_es2.glsl";
private final GlProgram glProgram;
private final ImmutableList<RgbMatrix> rgbMatrices;
private final boolean useHdr;
// TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrix The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public RgbMatrixProcessor(Context context, RgbMatrix rgbMatrix, boolean useHdr)
throws FrameProcessingException {
this(context, ImmutableList.of(rgbMatrix), useHdr);
}
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrices The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public RgbMatrixProcessor(Context context, ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr)
throws FrameProcessingException {
super(useHdr);
this.rgbMatrices = rgbMatrices;
this.useHdr = useHdr;
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
} catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
}
@Override
public Pair<Integer, Integer> configure(int inputWidth, int inputHeight) {
return Pair.create(inputWidth, inputHeight);
}
private static float[] createCompositeRgbaMatrixArray(
ImmutableList<RgbMatrix> rgbMatrices, boolean useHdr, long presentationTimeUs) {
float[] tempResultMatrix = new float[16];
float[] compositeRgbaMatrix = new float[16];
Matrix.setIdentityM(compositeRgbaMatrix, /* smOffset= */ 0);
for (int i = 0; i < rgbMatrices.size(); i++) {
Matrix.multiplyMM(
/* result= */ tempResultMatrix,
/* resultOffset= */ 0,
/* lhs= */ rgbMatrices.get(i).getMatrix(presentationTimeUs, useHdr),
/* lhsOffset= */ 0,
/* rhs= */ compositeRgbaMatrix,
/* rhsOffset= */ 0);
System.arraycopy(
/* src= */ tempResultMatrix,
/* srcPos= */ 0,
/* dest= */ compositeRgbaMatrix,
/* destPost= */ 0,
/* length= */ tempResultMatrix.length);
}
return compositeRgbaMatrix;
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
// TODO(b/239431666): Add caching for compacting Matrices.
float[] rgbMatrixArray =
createCompositeRgbaMatrixArray(rgbMatrices, useHdr, presentationTimeUs);
try {
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uColorMatrix", rgbMatrixArray);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment