Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
SDK
/
exoplayer
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Settings
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
672405af
authored
Sep 05, 2022
by
leonwind
Committed by
Marc Baechinger
Sep 30, 2022
Browse files
Options
_('Browse Files')
Download
Email Patches
Plain Diff
Merge RgbProcessor and MatrixTransformation.
PiperOrigin-RevId: 472325145
parent
3a2e0d37
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
207 additions
and
298 deletions
libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java
libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_es2.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl
libraries/effect/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl → libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java
libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTransformationProcessorPixelTest.java
View file @
672405af
...
@@ -28,7 +28,6 @@ import android.opengl.EGLSurface;
...
@@ -28,7 +28,6 @@ import android.opengl.EGLSurface;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.util.GlUtil
;
import
androidx.media3.common.util.GlUtil
;
import
androidx.test.ext.junit.runners.AndroidJUnit4
;
import
androidx.test.ext.junit.runners.AndroidJUnit4
;
import
com.google.common.collect.ImmutableList
;
import
java.io.IOException
;
import
java.io.IOException
;
import
org.checkerframework.checker.nullness.qual.MonotonicNonNull
;
import
org.checkerframework.checker.nullness.qual.MonotonicNonNull
;
import
org.junit.After
;
import
org.junit.After
;
...
@@ -97,8 +96,7 @@ public final class MatrixTransformationProcessorPixelTest {
...
@@ -97,8 +96,7 @@ public final class MatrixTransformationProcessorPixelTest {
Matrix
identityMatrix
=
new
Matrix
();
Matrix
identityMatrix
=
new
Matrix
();
MatrixTransformation
noEditsTransformation
=
(
long
presentationTimeUs
)
->
identityMatrix
;
MatrixTransformation
noEditsTransformation
=
(
long
presentationTimeUs
)
->
identityMatrix
;
matrixTransformationFrameProcessor
=
matrixTransformationFrameProcessor
=
MatrixTransformationProcessor
.
create
(
noEditsTransformation
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
context
,
ImmutableList
.
of
(
noEditsTransformation
),
/* useHdr= */
false
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
...
@@ -123,8 +121,7 @@ public final class MatrixTransformationProcessorPixelTest {
...
@@ -123,8 +121,7 @@ public final class MatrixTransformationProcessorPixelTest {
MatrixTransformation
translateRightTransformation
=
MatrixTransformation
translateRightTransformation
=
(
long
presentationTimeUs
)
->
translateRightMatrix
;
(
long
presentationTimeUs
)
->
translateRightMatrix
;
matrixTransformationFrameProcessor
=
matrixTransformationFrameProcessor
=
MatrixTransformationProcessor
.
create
(
translateRightTransformation
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
context
,
ImmutableList
.
of
(
translateRightTransformation
),
/* useHdr= */
false
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
TRANSLATE_RIGHT_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
TRANSLATE_RIGHT_PNG_ASSET_PATH
);
...
@@ -148,8 +145,7 @@ public final class MatrixTransformationProcessorPixelTest {
...
@@ -148,8 +145,7 @@ public final class MatrixTransformationProcessorPixelTest {
scaleNarrowMatrix
.
postScale
(.
5
f
,
1.2f
);
scaleNarrowMatrix
.
postScale
(.
5
f
,
1.2f
);
MatrixTransformation
scaleNarrowTransformation
=
(
long
presentationTimeUs
)
->
scaleNarrowMatrix
;
MatrixTransformation
scaleNarrowTransformation
=
(
long
presentationTimeUs
)
->
scaleNarrowMatrix
;
matrixTransformationFrameProcessor
=
matrixTransformationFrameProcessor
=
MatrixTransformationProcessor
.
create
(
scaleNarrowTransformation
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
context
,
ImmutableList
.
of
(
scaleNarrowTransformation
),
/* useHdr= */
false
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
SCALE_NARROW_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
SCALE_NARROW_PNG_ASSET_PATH
);
...
@@ -173,8 +169,7 @@ public final class MatrixTransformationProcessorPixelTest {
...
@@ -173,8 +169,7 @@ public final class MatrixTransformationProcessorPixelTest {
rotate90Matrix
.
postRotate
(
/* degrees= */
90
);
rotate90Matrix
.
postRotate
(
/* degrees= */
90
);
MatrixTransformation
rotate90Transformation
=
(
long
presentationTimeUs
)
->
rotate90Matrix
;
MatrixTransformation
rotate90Transformation
=
(
long
presentationTimeUs
)
->
rotate90Matrix
;
matrixTransformationFrameProcessor
=
matrixTransformationFrameProcessor
=
MatrixTransformationProcessor
.
create
(
rotate90Transformation
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
context
,
ImmutableList
.
of
(
rotate90Transformation
),
/* useHdr= */
false
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
matrixTransformationFrameProcessor
.
configure
(
width
,
height
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ROTATE_90_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ROTATE_90_PNG_ASSET_PATH
);
...
...
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
View file @
672405af
...
@@ -62,7 +62,7 @@ public final class RgbAdjustmentPixelTest {
...
@@ -62,7 +62,7 @@ public final class RgbAdjustmentPixelTest {
private
@MonotonicNonNull
EGLDisplay
eglDisplay
;
private
@MonotonicNonNull
EGLDisplay
eglDisplay
;
private
@MonotonicNonNull
EGLContext
eglContext
;
private
@MonotonicNonNull
EGLContext
eglContext
;
private
@MonotonicNonNull
SingleFrameGlTextureProcessor
rgbMatrix
Processor
;
private
@MonotonicNonNull
SingleFrameGlTextureProcessor
matrixTransformation
Processor
;
private
@MonotonicNonNull
EGLSurface
placeholderEglSurface
;
private
@MonotonicNonNull
EGLSurface
placeholderEglSurface
;
private
int
inputTexId
;
private
int
inputTexId
;
private
int
outputTexId
;
private
int
outputTexId
;
...
@@ -94,8 +94,8 @@ public final class RgbAdjustmentPixelTest {
...
@@ -94,8 +94,8 @@ public final class RgbAdjustmentPixelTest {
@After
@After
public
void
release
()
throws
GlUtil
.
GlException
,
FrameProcessingException
{
public
void
release
()
throws
GlUtil
.
GlException
,
FrameProcessingException
{
if
(
rgbMatrix
Processor
!=
null
)
{
if
(
matrixTransformation
Processor
!=
null
)
{
rgbMatrix
Processor
.
release
();
matrixTransformation
Processor
.
release
();
}
}
GlUtil
.
destroyEglContext
(
eglDisplay
,
eglContext
);
GlUtil
.
destroyEglContext
(
eglDisplay
,
eglContext
);
}
}
...
@@ -104,11 +104,13 @@ public final class RgbAdjustmentPixelTest {
...
@@ -104,11 +104,13 @@ public final class RgbAdjustmentPixelTest {
public
void
drawFrame_identityMatrix_leavesFrameUnchanged
()
throws
Exception
{
public
void
drawFrame_identityMatrix_leavesFrameUnchanged
()
throws
Exception
{
String
testId
=
"drawFrame_identityMatrix"
;
String
testId
=
"drawFrame_identityMatrix"
;
RgbMatrix
identityMatrix
=
new
RgbAdjustment
.
Builder
().
build
();
RgbMatrix
identityMatrix
=
new
RgbAdjustment
.
Builder
().
build
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
identityMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
identityMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -126,13 +128,15 @@ public final class RgbAdjustmentPixelTest {
...
@@ -126,13 +128,15 @@ public final class RgbAdjustmentPixelTest {
String
testId
=
"drawFrame_removeColors"
;
String
testId
=
"drawFrame_removeColors"
;
RgbMatrix
removeColorMatrix
=
RgbMatrix
removeColorMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
0
).
setGreenScale
(
0
).
setBlueScale
(
0
).
build
();
new
RgbAdjustment
.
Builder
().
setRedScale
(
0
).
setGreenScale
(
0
).
setBlueScale
(
0
).
build
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
removeColorMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
removeColorMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
Bitmap
expectedBitmap
=
BitmapTestUtil
.
createArgb8888BitmapWithSolidColor
(
BitmapTestUtil
.
createArgb8888BitmapWithSolidColor
(
outputSize
.
first
,
outputSize
.
second
,
Color
.
BLACK
);
outputSize
.
first
,
outputSize
.
second
,
Color
.
BLACK
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -149,11 +153,13 @@ public final class RgbAdjustmentPixelTest {
...
@@ -149,11 +153,13 @@ public final class RgbAdjustmentPixelTest {
public
void
drawFrame_redOnlyFilter_removeBlueAndGreenValues
()
throws
Exception
{
public
void
drawFrame_redOnlyFilter_removeBlueAndGreenValues
()
throws
Exception
{
String
testId
=
"drawFrame_redOnlyFilter"
;
String
testId
=
"drawFrame_redOnlyFilter"
;
RgbMatrix
redOnlyMatrix
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
setGreenScale
(
0
).
build
();
RgbMatrix
redOnlyMatrix
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
setGreenScale
(
0
).
build
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
redOnlyMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
redOnlyMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ONLY_RED_CHANNEL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ONLY_RED_CHANNEL_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -170,11 +176,13 @@ public final class RgbAdjustmentPixelTest {
...
@@ -170,11 +176,13 @@ public final class RgbAdjustmentPixelTest {
public
void
drawFrame_increaseRedChannel_producesBrighterAndRedderFrame
()
throws
Exception
{
public
void
drawFrame_increaseRedChannel_producesBrighterAndRedderFrame
()
throws
Exception
{
String
testId
=
"drawFrame_increaseRedChannel"
;
String
testId
=
"drawFrame_increaseRedChannel"
;
RgbMatrix
increaseRedMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
5
).
build
();
RgbMatrix
increaseRedMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
5
).
build
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
increaseRedMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
increaseRedMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INCREASE_RED_CHANNEL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INCREASE_RED_CHANNEL_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -192,12 +200,13 @@ public final class RgbAdjustmentPixelTest {
...
@@ -192,12 +200,13 @@ public final class RgbAdjustmentPixelTest {
String
testId
=
"drawFrame_increaseBrightness"
;
String
testId
=
"drawFrame_increaseBrightness"
;
RgbMatrix
increaseBrightnessMatrix
=
RgbMatrix
increaseBrightnessMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
5
).
setGreenScale
(
5
).
setBlueScale
(
5
).
build
();
new
RgbAdjustment
.
Builder
().
setRedScale
(
5
).
setGreenScale
(
5
).
setBlueScale
(
5
).
build
();
rgbMatrixProcessor
=
matrixTransformationProcessor
=
new
RgbMatrixProcessor
(
context
,
increaseBrightnessMatrix
,
/* useHdr = */
false
);
increaseBrightnessMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INCREASE_BRIGHTNESS_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INCREASE_BRIGHTNESS_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -216,15 +225,19 @@ public final class RgbAdjustmentPixelTest {
...
@@ -216,15 +225,19 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix
noRed
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
0
).
build
();
RgbMatrix
noRed
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
0
).
build
();
RgbMatrix
noGreen
=
new
RgbAdjustment
.
Builder
().
setGreenScale
(
0
).
build
();
RgbMatrix
noGreen
=
new
RgbAdjustment
.
Builder
().
setGreenScale
(
0
).
build
();
RgbMatrix
noBlue
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
build
();
RgbMatrix
noBlue
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
build
();
rgbMatrixProcessor
=
matrixTransformationProcessor
=
new
RgbMatrixProcessor
(
MatrixTransformationProcessor
.
create
(
context
,
ImmutableList
.
of
(
noRed
,
noGreen
,
noBlue
),
/* useHdr= */
false
);
context
,
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
/* matrixTransformations= */
ImmutableList
.
of
(),
/* rgbMatrices= */
ImmutableList
.
of
(
noRed
,
noGreen
,
noBlue
),
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
Bitmap
expectedBitmap
=
BitmapTestUtil
.
createArgb8888BitmapWithSolidColor
(
BitmapTestUtil
.
createArgb8888BitmapWithSolidColor
(
outputSize
.
first
,
outputSize
.
second
,
Color
.
BLACK
);
outputSize
.
first
,
outputSize
.
second
,
Color
.
BLACK
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -242,12 +255,17 @@ public final class RgbAdjustmentPixelTest {
...
@@ -242,12 +255,17 @@ public final class RgbAdjustmentPixelTest {
String
testId
=
"drawFrame_removeBlueAndGreenValuesInAChain"
;
String
testId
=
"drawFrame_removeBlueAndGreenValuesInAChain"
;
RgbMatrix
noGreen
=
new
RgbAdjustment
.
Builder
().
setGreenScale
(
0
).
build
();
RgbMatrix
noGreen
=
new
RgbAdjustment
.
Builder
().
setGreenScale
(
0
).
build
();
RgbMatrix
noBlue
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
build
();
RgbMatrix
noBlue
=
new
RgbAdjustment
.
Builder
().
setBlueScale
(
0
).
build
();
rgbMatrixProcessor
=
matrixTransformationProcessor
=
new
RgbMatrixProcessor
(
context
,
ImmutableList
.
of
(
noGreen
,
noBlue
),
/* useHdr= */
false
);
MatrixTransformationProcessor
.
create
(
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
context
,
/* matrixTransformations= */
ImmutableList
.
of
(),
/* rgbMatrices= */
ImmutableList
.
of
(
noGreen
,
noBlue
),
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ONLY_RED_CHANNEL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ONLY_RED_CHANNEL_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -267,15 +285,17 @@ public final class RgbAdjustmentPixelTest {
...
@@ -267,15 +285,17 @@ public final class RgbAdjustmentPixelTest {
RgbMatrix
scaleRedMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
redScale
).
build
();
RgbMatrix
scaleRedMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
redScale
).
build
();
RgbMatrix
scaleRedByInverseMatrix
=
RgbMatrix
scaleRedByInverseMatrix
=
new
RgbAdjustment
.
Builder
().
setRedScale
(
1
/
redScale
).
build
();
new
RgbAdjustment
.
Builder
().
setRedScale
(
1
/
redScale
).
build
();
rgbMatrix
Processor
=
matrixTransformation
Processor
=
new
RgbMatrixProcessor
(
MatrixTransformationProcessor
.
create
(
context
,
context
,
ImmutableList
.
of
(
scaleRedMatrix
,
scaleRedByInverseMatrix
),
/* matrixTransformations= */
ImmutableList
.
of
(),
/* rgbMatrices= */
ImmutableList
.
of
(
scaleRedMatrix
,
scaleRedByInverseMatrix
),
/* useHdr= */
false
);
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
ORIGINAL_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
...
libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java
View file @
672405af
...
@@ -58,7 +58,7 @@ public final class RgbFilterPixelTest {
...
@@ -58,7 +58,7 @@ public final class RgbFilterPixelTest {
private
@MonotonicNonNull
EGLDisplay
eglDisplay
;
private
@MonotonicNonNull
EGLDisplay
eglDisplay
;
private
@MonotonicNonNull
EGLContext
eglContext
;
private
@MonotonicNonNull
EGLContext
eglContext
;
private
@MonotonicNonNull
SingleFrameGlTextureProcessor
rgbMatrix
Processor
;
private
@MonotonicNonNull
SingleFrameGlTextureProcessor
matrixTransformation
Processor
;
private
@MonotonicNonNull
EGLSurface
placeholderEglSurface
;
private
@MonotonicNonNull
EGLSurface
placeholderEglSurface
;
private
int
inputTexId
;
private
int
inputTexId
;
private
int
outputTexId
;
private
int
outputTexId
;
...
@@ -90,8 +90,8 @@ public final class RgbFilterPixelTest {
...
@@ -90,8 +90,8 @@ public final class RgbFilterPixelTest {
@After
@After
public
void
release
()
throws
GlUtil
.
GlException
,
FrameProcessingException
{
public
void
release
()
throws
GlUtil
.
GlException
,
FrameProcessingException
{
if
(
rgbMatrix
Processor
!=
null
)
{
if
(
matrixTransformation
Processor
!=
null
)
{
rgbMatrix
Processor
.
release
();
matrixTransformation
Processor
.
release
();
}
}
GlUtil
.
destroyEglContext
(
eglDisplay
,
eglContext
);
GlUtil
.
destroyEglContext
(
eglDisplay
,
eglContext
);
}
}
...
@@ -100,11 +100,13 @@ public final class RgbFilterPixelTest {
...
@@ -100,11 +100,13 @@ public final class RgbFilterPixelTest {
public
void
drawFrame_grayscale_producesGrayscaleImage
()
throws
Exception
{
public
void
drawFrame_grayscale_producesGrayscaleImage
()
throws
Exception
{
String
testId
=
"drawFrame_grayscale"
;
String
testId
=
"drawFrame_grayscale"
;
RgbMatrix
grayscaleMatrix
=
RgbFilter
.
createGrayscaleFilter
();
RgbMatrix
grayscaleMatrix
=
RgbFilter
.
createGrayscaleFilter
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
grayscaleMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
grayscaleMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
GRAYSCALE_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
GRAYSCALE_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
@@ -121,11 +123,13 @@ public final class RgbFilterPixelTest {
...
@@ -121,11 +123,13 @@ public final class RgbFilterPixelTest {
public
void
drawFrame_inverted_producesInvertedFrame
()
throws
Exception
{
public
void
drawFrame_inverted_producesInvertedFrame
()
throws
Exception
{
String
testId
=
"drawFrame_inverted"
;
String
testId
=
"drawFrame_inverted"
;
RgbMatrix
invertedMatrix
=
RgbFilter
.
createInvertedFilter
();
RgbMatrix
invertedMatrix
=
RgbFilter
.
createInvertedFilter
();
rgbMatrixProcessor
=
new
RgbMatrixProcessor
(
context
,
invertedMatrix
,
/* useHdr= */
false
);
matrixTransformationProcessor
=
Pair
<
Integer
,
Integer
>
outputSize
=
rgbMatrixProcessor
.
configure
(
inputWidth
,
inputHeight
);
invertedMatrix
.
toGlTextureProcessor
(
context
,
/* useHdr= */
false
);
Pair
<
Integer
,
Integer
>
outputSize
=
matrixTransformationProcessor
.
configure
(
inputWidth
,
inputHeight
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INVERT_PNG_ASSET_PATH
);
Bitmap
expectedBitmap
=
BitmapTestUtil
.
readBitmap
(
INVERT_PNG_ASSET_PATH
);
rgbMatrix
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
matrixTransformation
Processor
.
drawFrame
(
inputTexId
,
/* presentationTimeUs= */
0
);
Bitmap
actualBitmap
=
Bitmap
actualBitmap
=
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
BitmapTestUtil
.
createArgb8888BitmapFromCurrentGlFramebuffer
(
outputSize
.
first
,
outputSize
.
second
);
outputSize
.
first
,
outputSize
.
second
);
...
...
libraries/effect/src/main/assets/shaders/fragment_shader_copy_es2.glsl
deleted
100644 → 0
View file @
3a2e0d37
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler,
// copying from this texture to the current output.
precision
mediump
float
;
uniform
sampler2D
uTexSampler
;
varying
vec2
vTexSamplingCoord
;
void
main
()
{
gl_FragColor
=
texture2D
(
uTexSampler
,
vTexSamplingCoord
);
}
libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl
View file @
672405af
...
@@ -15,7 +15,8 @@
...
@@ -15,7 +15,8 @@
// ES 3 fragment shader that:
// ES 3 fragment shader that:
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
// uTexSampler,
// uTexSampler, and applies a 4x4 RGB color matrix to change the pixel
// colors,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
// and
// and
// 3. copies this converted texture color to the current output.
// 3. copies this converted texture color to the current output.
...
@@ -28,6 +29,7 @@ out vec4 outColor;
...
@@ -28,6 +29,7 @@ out vec4 outColor;
// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
uniform
int
uOetfColorTransfer
;
uniform
int
uOetfColorTransfer
;
uniform
mat3
uColorTransform
;
uniform
mat3
uColorTransform
;
uniform
mat4
uRgbMatrix
;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization.
// to noticeable quantization.
...
@@ -83,5 +85,6 @@ highp vec3 getElectricalColor(highp vec3 linearColor) {
...
@@ -83,5 +85,6 @@ highp vec3 getElectricalColor(highp vec3 linearColor) {
void
main
()
{
void
main
()
{
vec4
inputColor
=
texture
(
uTexSampler
,
vTexSamplingCoord
);
vec4
inputColor
=
texture
(
uTexSampler
,
vTexSamplingCoord
);
outColor
=
vec4
(
getElectricalColor
(
inputColor
.
rgb
),
inputColor
.
a
);
vec4
transformedColors
=
uRgbMatrix
*
vec4
(
inputColor
.
rgb
,
1
);
outColor
=
vec4
(
getElectricalColor
(
transformedColors
.
rgb
),
inputColor
.
a
);
}
}
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_es2.glsl
View file @
672405af
...
@@ -19,11 +19,11 @@
...
@@ -19,11 +19,11 @@
precision
mediump
float
;
precision
mediump
float
;
uniform
sampler2D
uTexSampler
;
uniform
sampler2D
uTexSampler
;
uniform
mat4
u
Color
Matrix
;
uniform
mat4
u
Rgb
Matrix
;
varying
vec2
vTexSamplingCoord
;
varying
vec2
vTexSamplingCoord
;
void
main
()
{
void
main
()
{
vec4
inputColor
=
texture2D
(
uTexSampler
,
vTexSamplingCoord
);
vec4
inputColor
=
texture2D
(
uTexSampler
,
vTexSamplingCoord
);
gl_FragColor
=
u
Color
Matrix
*
vec4
(
inputColor
.
rgb
,
1
);
gl_FragColor
=
u
Rgb
Matrix
*
vec4
(
inputColor
.
rgb
,
1
);
gl_FragColor
.
a
=
inputColor
.
a
;
gl_FragColor
.
a
=
inputColor
.
a
;
}
}
libraries/effect/src/main/assets/shaders/fragment_shader_
copy
_external_es2.glsl
→
libraries/effect/src/main/assets/shaders/fragment_shader_
transformation
_external_es2.glsl
View file @
672405af
...
@@ -14,12 +14,17 @@
...
@@ -14,12 +14,17 @@
// limitations under the License.
// limitations under the License.
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// ES 2 fragment shader that samples from an external texture with uTexSampler,
// copying from this texture to the current output.
// copying from this texture to the current output while applying a 4x4 RGB
// color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
#extension GL_OES_EGL_image_external : require
precision
mediump
float
;
precision
mediump
float
;
uniform
samplerExternalOES
uTexSampler
;
uniform
samplerExternalOES
uTexSampler
;
uniform
mat4
uRgbMatrix
;
varying
vec2
vTexSamplingCoord
;
varying
vec2
vTexSamplingCoord
;
void
main
()
{
void
main
()
{
gl_FragColor
=
texture2D
(
uTexSampler
,
vTexSamplingCoord
);
vec4
inputColor
=
texture2D
(
uTexSampler
,
vTexSamplingCoord
);
gl_FragColor
=
uRgbMatrix
*
vec4
(
inputColor
.
rgb
,
1
);
gl_FragColor
.
a
=
inputColor
.
a
;
}
}
libraries/effect/src/main/assets/shaders/fragment_shader_
copy
_external_yuv_es3.glsl
→
libraries/effect/src/main/assets/shaders/fragment_shader_
transformation
_external_yuv_es3.glsl
View file @
672405af
...
@@ -23,13 +23,15 @@
...
@@ -23,13 +23,15 @@
// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical
// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical
// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
// intermediate shaders by applying the HLG or PQ EOTF.
// intermediate shaders by applying the HLG or PQ EOTF.
// 4. Copies this converted texture color to the current output, with alpha = 1.
// 4. Copies this converted texture color to the current output, with alpha = 1,
// while applying a 4x4 RGB color matrix to change the pixel colors.
#extension GL_OES_EGL_image_external : require
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require
#extension GL_EXT_YUV_target : require
precision
mediump
float
;
precision
mediump
float
;
uniform
__samplerExternal2DY2YEXT
uTexSampler
;
uniform
__samplerExternal2DY2YEXT
uTexSampler
;
uniform
mat3
uYuvToRgbColorTransform
;
uniform
mat3
uYuvToRgbColorTransform
;
uniform
mat4
uRgbMatrix
;
// C.java#ColorTransfer value.
// C.java#ColorTransfer value.
uniform
int
uEotfColorTransfer
;
uniform
int
uEotfColorTransfer
;
in
vec2
vTexSamplingCoord
;
in
vec2
vTexSamplingCoord
;
...
@@ -101,5 +103,5 @@ vec3 yuvToRgb(vec3 yuv) {
...
@@ -101,5 +103,5 @@ vec3 yuvToRgb(vec3 yuv) {
void
main
()
{
void
main
()
{
vec3
srcYuv
=
texture
(
uTexSampler
,
vTexSamplingCoord
).
xyz
;
vec3
srcYuv
=
texture
(
uTexSampler
,
vTexSamplingCoord
).
xyz
;
vec3
rgb
=
yuvToRgb
(
srcYuv
);
vec3
rgb
=
yuvToRgb
(
srcYuv
);
outColor
=
vec4
(
getOpticalColor
(
rgb
),
1
.
0
);
outColor
=
uRgbMatrix
*
vec4
(
getOpticalColor
(
rgb
),
1
.
0
);
}
}
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
View file @
672405af
...
@@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...
@@ -66,6 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private
final
Context
context
;
private
final
Context
context
;
private
final
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
;
private
final
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
;
private
final
ImmutableList
<
RgbMatrix
>
rgbMatrices
;
private
final
EGLDisplay
eglDisplay
;
private
final
EGLDisplay
eglDisplay
;
private
final
EGLContext
eglContext
;
private
final
EGLContext
eglContext
;
private
final
DebugViewProvider
debugViewProvider
;
private
final
DebugViewProvider
debugViewProvider
;
...
@@ -100,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...
@@ -100,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLDisplay
eglDisplay
,
EGLDisplay
eglDisplay
,
EGLContext
eglContext
,
EGLContext
eglContext
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
FrameProcessor
.
Listener
frameProcessorListener
,
FrameProcessor
.
Listener
frameProcessorListener
,
DebugViewProvider
debugViewProvider
,
DebugViewProvider
debugViewProvider
,
boolean
sampleFromExternalTexture
,
boolean
sampleFromExternalTexture
,
...
@@ -107,6 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...
@@ -107,6 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
boolean
releaseFramesAutomatically
)
{
boolean
releaseFramesAutomatically
)
{
this
.
context
=
context
;
this
.
context
=
context
;
this
.
matrixTransformations
=
matrixTransformations
;
this
.
matrixTransformations
=
matrixTransformations
;
this
.
rgbMatrices
=
rgbMatrices
;
this
.
eglDisplay
=
eglDisplay
;
this
.
eglDisplay
=
eglDisplay
;
this
.
eglContext
=
eglContext
;
this
.
eglContext
=
eglContext
;
this
.
debugViewProvider
=
debugViewProvider
;
this
.
debugViewProvider
=
debugViewProvider
;
...
@@ -374,11 +377,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...
@@ -374,11 +377,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if
(
sampleFromExternalTexture
)
{
if
(
sampleFromExternalTexture
)
{
matrixTransformationProcessor
=
matrixTransformationProcessor
=
MatrixTransformationProcessor
.
createWithExternalSamplerApplyingEotfThenOetf
(
MatrixTransformationProcessor
.
createWithExternalSamplerApplyingEotfThenOetf
(
context
,
expandedMatrixTransformations
,
colorInfo
);
context
,
expandedMatrixTransformations
,
rgbMatrices
,
colorInfo
);
}
else
{
}
else
{
matrixTransformationProcessor
=
matrixTransformationProcessor
=
MatrixTransformationProcessor
.
createApplyingOetf
(
MatrixTransformationProcessor
.
createApplyingOetf
(
context
,
expandedMatrixTransformations
,
colorInfo
);
context
,
expandedMatrixTransformations
,
rgbMatrices
,
colorInfo
);
}
}
matrixTransformationProcessor
.
setTextureTransformMatrix
(
textureTransformMatrix
);
matrixTransformationProcessor
.
setTextureTransformMatrix
(
textureTransformMatrix
);
...
...
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View file @
672405af
...
@@ -184,8 +184,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
...
@@ -184,8 +184,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
new
ImmutableList
.
Builder
<>();
new
ImmutableList
.
Builder
<>();
ImmutableList
.
Builder
<
GlMatrixTransformation
>
matrixTransformationListBuilder
=
ImmutableList
.
Builder
<
GlMatrixTransformation
>
matrixTransformationListBuilder
=
new
ImmutableList
.
Builder
<>();
new
ImmutableList
.
Builder
<>();
ImmutableList
.
Builder
<
RgbMatrix
>
rgbaMatrixTransformationListBuilder
=
ImmutableList
.
Builder
<
RgbMatrix
>
rgbMatrixListBuilder
=
new
ImmutableList
.
Builder
<>();
new
ImmutableList
.
Builder
<>();
boolean
sampleFromExternalTexture
=
true
;
boolean
sampleFromExternalTexture
=
true
;
for
(
int
i
=
0
;
i
<
effects
.
size
();
i
++)
{
for
(
int
i
=
0
;
i
<
effects
.
size
();
i
++)
{
Effect
effect
=
effects
.
get
(
i
);
Effect
effect
=
effects
.
get
(
i
);
...
@@ -200,62 +199,39 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
...
@@ -200,62 +199,39 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
continue
;
continue
;
}
}
if
(
glEffect
instanceof
RgbMatrix
)
{
if
(
glEffect
instanceof
RgbMatrix
)
{
rgb
aMatrixTransformation
ListBuilder
.
add
((
RgbMatrix
)
glEffect
);
rgb
Matrix
ListBuilder
.
add
((
RgbMatrix
)
glEffect
);
continue
;
continue
;
}
}
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
=
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
=
matrixTransformationListBuilder
.
build
();
matrixTransformationListBuilder
.
build
();
if
(!
matrixTransformations
.
isEmpty
()
||
sampleFromExternalTexture
)
{
ImmutableList
<
RgbMatrix
>
rgbMatrices
=
rgbMatrixListBuilder
.
build
();
if
(!
matrixTransformations
.
isEmpty
()
||
!
rgbMatrices
.
isEmpty
()
||
sampleFromExternalTexture
)
{
MatrixTransformationProcessor
matrixTransformationProcessor
;
MatrixTransformationProcessor
matrixTransformationProcessor
;
if
(
sampleFromExternalTexture
)
{
if
(
sampleFromExternalTexture
)
{
matrixTransformationProcessor
=
matrixTransformationProcessor
=
MatrixTransformationProcessor
.
createWithExternalSamplerApplyingEotf
(
MatrixTransformationProcessor
.
createWithExternalSamplerApplyingEotf
(
context
,
matrixTransformations
,
colorInfo
);
context
,
matrixTransformations
,
rgbMatrices
,
colorInfo
);
}
else
{
}
else
{
matrixTransformationProcessor
=
matrixTransformationProcessor
=
MatrixTransformationProcessor
.
create
(
MatrixTransformationProcessor
.
create
(
context
,
matrixTransformations
,
ColorInfo
.
isTransferHdr
(
colorInfo
));
context
,
matrixTransformations
,
rgbMatrices
,
ColorInfo
.
isTransferHdr
(
colorInfo
));
}
}
textureProcessorListBuilder
.
add
(
matrixTransformationProcessor
);
textureProcessorListBuilder
.
add
(
matrixTransformationProcessor
);
matrixTransformationListBuilder
=
new
ImmutableList
.
Builder
<>();
matrixTransformationListBuilder
=
new
ImmutableList
.
Builder
<>();
rgbMatrixListBuilder
=
new
ImmutableList
.
Builder
<>();
sampleFromExternalTexture
=
false
;
sampleFromExternalTexture
=
false
;
}
}
ImmutableList
<
RgbMatrix
>
rgbaMatrixTransformations
=
rgbaMatrixTransformationListBuilder
.
build
();
if
(!
rgbaMatrixTransformations
.
isEmpty
())
{
textureProcessorListBuilder
.
add
(
new
RgbMatrixProcessor
(
context
,
rgbaMatrixTransformations
,
ColorInfo
.
isTransferHdr
(
colorInfo
)));
rgbaMatrixTransformationListBuilder
=
new
ImmutableList
.
Builder
<>();
}
textureProcessorListBuilder
.
add
(
textureProcessorListBuilder
.
add
(
glEffect
.
toGlTextureProcessor
(
context
,
ColorInfo
.
isTransferHdr
(
colorInfo
)));
glEffect
.
toGlTextureProcessor
(
context
,
ColorInfo
.
isTransferHdr
(
colorInfo
)));
}
}
ImmutableList
<
RgbMatrix
>
rgbaMatrixTransformations
=
rgbaMatrixTransformationListBuilder
.
build
();
if
(!
rgbaMatrixTransformations
.
isEmpty
())
{
// Add a MatrixTransformationProcessor if none yet exists for sampling from an external
// texture.
if
(
sampleFromExternalTexture
)
{
// TODO(b/239757183): Remove the unnecessary MatrixTransformationProcessor after it got
// merged with RgbMatrixProcessor.
textureProcessorListBuilder
.
add
(
MatrixTransformationProcessor
.
createWithExternalSamplerApplyingEotf
(
context
,
/* matrixTransformations= */
ImmutableList
.
of
(),
colorInfo
));
sampleFromExternalTexture
=
false
;
}
textureProcessorListBuilder
.
add
(
new
RgbMatrixProcessor
(
context
,
rgbaMatrixTransformations
,
ColorInfo
.
isTransferHdr
(
colorInfo
)));
}
textureProcessorListBuilder
.
add
(
textureProcessorListBuilder
.
add
(
new
FinalMatrixTransformationProcessorWrapper
(
new
FinalMatrixTransformationProcessorWrapper
(
context
,
context
,
eglDisplay
,
eglDisplay
,
eglContext
,
eglContext
,
matrixTransformationListBuilder
.
build
(),
matrixTransformationListBuilder
.
build
(),
rgbMatrixListBuilder
.
build
(),
listener
,
listener
,
debugViewProvider
,
debugViewProvider
,
sampleFromExternalTexture
,
sampleFromExternalTexture
,
...
...
libraries/effect/src/main/java/androidx/media3/effect/GlMatrixTransformation.java
View file @
672405af
...
@@ -56,6 +56,9 @@ public interface GlMatrixTransformation extends GlEffect {
...
@@ -56,6 +56,9 @@ public interface GlMatrixTransformation extends GlEffect {
default
SingleFrameGlTextureProcessor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
default
SingleFrameGlTextureProcessor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
return
MatrixTransformationProcessor
.
create
(
return
MatrixTransformationProcessor
.
create
(
context
,
/* matrixTransformations= */
ImmutableList
.
of
(
this
),
useHdr
);
context
,
/* matrixTransformations= */
ImmutableList
.
of
(
this
),
/* rgbMatrices= */
ImmutableList
.
of
(),
useHdr
);
}
}
}
}
libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
View file @
672405af
...
@@ -34,17 +34,22 @@ import java.io.IOException;
...
@@ -34,17 +34,22 @@ import java.io.IOException;
import
java.util.Arrays
;
import
java.util.Arrays
;
/**
/**
* Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into
* Applies a sequence of {@link MatrixTransformation MatrixTransformations} in the vertex shader and
* an output frame based on their locations after applying the sequence of transformation matrices.
* a sequence of {@link RgbMatrix RgbMatrices} in the fragment shader. Copies input pixels into an
* output frame based on their locations after applying the sequence of transformation matrices.
*
*
* <p>Operations are done on normalized device coordinates (-1 to 1 on x, y, and z axes).
* <p>{@link MatrixTransformation} operations are done on normalized device coordinates (-1 to 1 on
* Transformed vertices that are moved outside of this range after any of the transformation
* x, y, and z axes). Transformed vertices that are moved outside of this range after any of the
* matrices are clipped to the NDC range.
* transformation matrices are clipped to the NDC range.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space (e.g. BT.709 for SDR). Intermediate results are not clamped.
*
*
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
* <p>The background color of the output frame will be (r=0, g=0, b=0, a=0).
*
*
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/
*/
// TODO(b/239757183): Rename Matrix to a more generic name.
@UnstableApi
@UnstableApi
@SuppressWarnings
(
"FunctionalInterfaceClash"
)
// b/228192298
@SuppressWarnings
(
"FunctionalInterfaceClash"
)
// b/228192298
/* package */
final
class
MatrixTransformationProcessor
extends
SingleFrameGlTextureProcessor
/* package */
final
class
MatrixTransformationProcessor
extends
SingleFrameGlTextureProcessor
...
@@ -54,13 +59,14 @@ import java.util.Arrays;
...
@@ -54,13 +59,14 @@ import java.util.Arrays;
"shaders/vertex_shader_transformation_es2.glsl"
;
"shaders/vertex_shader_transformation_es2.glsl"
;
private
static
final
String
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
=
private
static
final
String
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
=
"shaders/vertex_shader_transformation_es3.glsl"
;
"shaders/vertex_shader_transformation_es3.glsl"
;
private
static
final
String
FRAGMENT_SHADER_COPY_PATH
=
"shaders/fragment_shader_copy_es2.glsl"
;
private
static
final
String
FRAGMENT_SHADER_TRANSFORMATION_PATH
=
"shaders/fragment_shader_transformation_es2.glsl"
;
private
static
final
String
FRAGMENT_SHADER_OETF_ES3_PATH
=
private
static
final
String
FRAGMENT_SHADER_OETF_ES3_PATH
=
"shaders/fragment_shader_oetf_es3.glsl"
;
"shaders/fragment_shader_oetf_es3.glsl"
;
private
static
final
String
FRAGMENT_SHADER_
COPY
_EXTERNAL_PATH
=
private
static
final
String
FRAGMENT_SHADER_
TRANSFORMATION
_EXTERNAL_PATH
=
"shaders/fragment_shader_
copy
_external_es2.glsl"
;
"shaders/fragment_shader_
transformation
_external_es2.glsl"
;
private
static
final
String
FRAGMENT_SHADER_
COPY
_EXTERNAL_YUV_ES3_PATH
=
private
static
final
String
FRAGMENT_SHADER_
TRANSFORMATION
_EXTERNAL_YUV_ES3_PATH
=
"shaders/fragment_shader_
copy
_external_yuv_es3.glsl"
;
"shaders/fragment_shader_
transformation
_external_yuv_es3.glsl"
;
private
static
final
ImmutableList
<
float
[]>
NDC_SQUARE
=
private
static
final
ImmutableList
<
float
[]>
NDC_SQUARE
=
ImmutableList
.
of
(
ImmutableList
.
of
(
new
float
[]
{-
1
,
-
1
,
0
,
1
},
new
float
[]
{-
1
,
-
1
,
0
,
1
},
...
@@ -84,6 +90,10 @@ import java.util.Arrays;
...
@@ -84,6 +90,10 @@ import java.util.Arrays;
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
private
final
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
;
private
final
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
;
/** The {@link RgbMatrix RgbMatrices} to apply. */
private
final
ImmutableList
<
RgbMatrix
>
rgbMatrices
;
/** Whether the frame is in HDR or not. */
private
final
boolean
useHdr
;
/**
/**
* The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations}
* The transformation matrices provided by the {@link MatrixTransformation MatrixTransformations}
* for the most recent frame.
* for the most recent frame.
...
@@ -117,19 +127,25 @@ import java.util.Arrays;
...
@@ -117,19 +127,25 @@ import java.util.Arrays;
*
*
* @param context The {@link Context}.
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether input and output colors are HDR.
* @param useHdr Whether input and output colors are HDR.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
* operation fails or is unsupported.
*/
*/
public
static
MatrixTransformationProcessor
create
(
public
static
MatrixTransformationProcessor
create
(
Context
context
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
boolean
useHdr
)
Context
context
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
boolean
useHdr
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
GlProgram
glProgram
=
GlProgram
glProgram
=
createGlProgram
(
context
,
VERTEX_SHADER_TRANSFORMATION_PATH
,
FRAGMENT_SHADER_COPY_PATH
);
createGlProgram
(
context
,
VERTEX_SHADER_TRANSFORMATION_PATH
,
FRAGMENT_SHADER_TRANSFORMATION_PATH
);
// No transfer functions needed, because input and output are both optical colors.
// No transfer functions needed, because input and output are both optical colors.
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
useHdr
);
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
rgbMatrices
,
useHdr
);
}
}
/**
/**
...
@@ -148,7 +164,9 @@ import java.util.Arrays;
...
@@ -148,7 +164,9 @@ import java.util.Arrays;
*
*
* @param context The {@link Context}.
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
* operation fails or is unsupported.
...
@@ -156,13 +174,16 @@ import java.util.Arrays;
...
@@ -156,13 +174,16 @@ import java.util.Arrays;
public
static
MatrixTransformationProcessor
createWithExternalSamplerApplyingEotf
(
public
static
MatrixTransformationProcessor
createWithExternalSamplerApplyingEotf
(
Context
context
,
Context
context
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
ColorInfo
electricalColorInfo
)
ColorInfo
electricalColorInfo
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
String
vertexShaderFilePath
=
String
vertexShaderFilePath
=
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
String
fragmentShaderFilePath
=
String
fragmentShaderFilePath
=
useHdr
?
FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
:
FRAGMENT_SHADER_COPY_EXTERNAL_PATH
;
useHdr
?
FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
:
FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH
;
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
// TODO(b/241902517): Implement gamma transfer functions.
// TODO(b/241902517): Implement gamma transfer functions.
...
@@ -184,7 +205,7 @@ import java.util.Arrays;
...
@@ -184,7 +205,7 @@ import java.util.Arrays;
glProgram
.
setIntUniform
(
"uEotfColorTransfer"
,
colorTransfer
);
glProgram
.
setIntUniform
(
"uEotfColorTransfer"
,
colorTransfer
);
}
}
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
useHdr
);
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
rgbMatrices
,
useHdr
);
}
}
/**
/**
...
@@ -199,7 +220,9 @@ import java.util.Arrays;
...
@@ -199,7 +220,9 @@ import java.util.Arrays;
*
*
* @param context The {@link Context}.
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @param electricalColorInfo The electrical {@link ColorInfo} describing output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
* operation fails or is unsupported.
...
@@ -207,13 +230,14 @@ import java.util.Arrays;
...
@@ -207,13 +230,14 @@ import java.util.Arrays;
public
static
MatrixTransformationProcessor
createApplyingOetf
(
public
static
MatrixTransformationProcessor
createApplyingOetf
(
Context
context
,
Context
context
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
ColorInfo
electricalColorInfo
)
ColorInfo
electricalColorInfo
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
String
vertexShaderFilePath
=
String
vertexShaderFilePath
=
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
String
fragmentShaderFilePath
=
String
fragmentShaderFilePath
=
useHdr
?
FRAGMENT_SHADER_OETF_ES3_PATH
:
FRAGMENT_SHADER_
COPY
_PATH
;
useHdr
?
FRAGMENT_SHADER_OETF_ES3_PATH
:
FRAGMENT_SHADER_
TRANSFORMATION
_PATH
;
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
// TODO(b/241902517): Implement gamma transfer functions.
// TODO(b/241902517): Implement gamma transfer functions.
...
@@ -224,7 +248,7 @@ import java.util.Arrays;
...
@@ -224,7 +248,7 @@ import java.util.Arrays;
glProgram
.
setIntUniform
(
"uOetfColorTransfer"
,
colorTransfer
);
glProgram
.
setIntUniform
(
"uOetfColorTransfer"
,
colorTransfer
);
}
}
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
useHdr
);
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
rgbMatrices
,
useHdr
);
}
}
/**
/**
...
@@ -239,7 +263,9 @@ import java.util.Arrays;
...
@@ -239,7 +263,9 @@ import java.util.Arrays;
*
*
* @param context The {@link Context}.
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @param electricalColorInfo The electrical {@link ColorInfo} describing input and output colors.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
* operation fails or is unsupported.
...
@@ -247,13 +273,16 @@ import java.util.Arrays;
...
@@ -247,13 +273,16 @@ import java.util.Arrays;
public
static
MatrixTransformationProcessor
createWithExternalSamplerApplyingEotfThenOetf
(
public
static
MatrixTransformationProcessor
createWithExternalSamplerApplyingEotfThenOetf
(
Context
context
,
Context
context
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
ColorInfo
electricalColorInfo
)
ColorInfo
electricalColorInfo
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
boolean
useHdr
=
ColorInfo
.
isTransferHdr
(
electricalColorInfo
);
String
vertexShaderFilePath
=
String
vertexShaderFilePath
=
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
useHdr
?
VERTEX_SHADER_TRANSFORMATION_ES3_PATH
:
VERTEX_SHADER_TRANSFORMATION_PATH
;
String
fragmentShaderFilePath
=
String
fragmentShaderFilePath
=
useHdr
?
FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
:
FRAGMENT_SHADER_COPY_EXTERNAL_PATH
;
useHdr
?
FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
:
FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH
;
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
GlProgram
glProgram
=
createGlProgram
(
context
,
vertexShaderFilePath
,
fragmentShaderFilePath
);
// TODO(b/241902517): Implement gamma transfer functions.
// TODO(b/241902517): Implement gamma transfer functions.
...
@@ -273,7 +302,7 @@ import java.util.Arrays;
...
@@ -273,7 +302,7 @@ import java.util.Arrays;
glProgram
.
setIntUniform
(
"uEotfColorTransfer"
,
Format
.
NO_VALUE
);
glProgram
.
setIntUniform
(
"uEotfColorTransfer"
,
Format
.
NO_VALUE
);
}
}
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
useHdr
);
return
new
MatrixTransformationProcessor
(
glProgram
,
matrixTransformations
,
rgbMatrices
,
useHdr
);
}
}
/**
/**
...
@@ -281,17 +310,22 @@ import java.util.Arrays;
...
@@ -281,17 +310,22 @@ import java.util.Arrays;
*
*
* @param glProgram The {@link GlProgram}.
* @param glProgram The {@link GlProgram}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
* EXT_YUV_target} OpenGL extension.
*/
*/
private
MatrixTransformationProcessor
(
private
MatrixTransformationProcessor
(
GlProgram
glProgram
,
GlProgram
glProgram
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
GlMatrixTransformation
>
matrixTransformations
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
boolean
useHdr
)
{
boolean
useHdr
)
{
super
(
useHdr
);
super
(
useHdr
);
this
.
glProgram
=
glProgram
;
this
.
glProgram
=
glProgram
;
this
.
matrixTransformations
=
matrixTransformations
;
this
.
matrixTransformations
=
matrixTransformations
;
this
.
rgbMatrices
=
rgbMatrices
;
this
.
useHdr
=
useHdr
;
transformationMatrixCache
=
new
float
[
matrixTransformations
.
size
()][
16
];
transformationMatrixCache
=
new
float
[
matrixTransformations
.
size
()][
16
];
compositeTransformationMatrix
=
new
float
[
16
];
compositeTransformationMatrix
=
new
float
[
16
];
...
@@ -333,11 +367,13 @@ import java.util.Arrays;
...
@@ -333,11 +367,13 @@ import java.util.Arrays;
if
(
visiblePolygon
.
size
()
<
3
)
{
if
(
visiblePolygon
.
size
()
<
3
)
{
return
;
// Need at least three visible vertices for a triangle.
return
;
// Need at least three visible vertices for a triangle.
}
}
float
[]
compositeRgbMatrix
=
createCompositeRgbaMatrixArray
(
rgbMatrices
,
useHdr
,
presentationTimeUs
);
try
{
try
{
glProgram
.
use
();
glProgram
.
use
();
glProgram
.
setSamplerTexIdUniform
(
"uTexSampler"
,
inputTexId
,
/* texUnitIndex= */
0
);
glProgram
.
setSamplerTexIdUniform
(
"uTexSampler"
,
inputTexId
,
/* texUnitIndex= */
0
);
glProgram
.
setFloatsUniform
(
"uTransformationMatrix"
,
compositeTransformationMatrix
);
glProgram
.
setFloatsUniform
(
"uTransformationMatrix"
,
compositeTransformationMatrix
);
glProgram
.
setFloatsUniform
(
"uRgbMatrix"
,
compositeRgbMatrix
);
glProgram
.
setBufferAttribute
(
glProgram
.
setBufferAttribute
(
"aFramePosition"
,
"aFramePosition"
,
GlUtil
.
createVertexBuffer
(
visiblePolygon
),
GlUtil
.
createVertexBuffer
(
visiblePolygon
),
...
@@ -425,4 +461,31 @@ import java.util.Arrays;
...
@@ -425,4 +461,31 @@ import java.util.Arrays;
}
}
return
matrixChanged
;
return
matrixChanged
;
}
}
// TODO(b/239757183): Add caching for RgbMatrix and refactor RgbMatrix and MatrixTransformation
// composing.
private
static
float
[]
createCompositeRgbaMatrixArray
(
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
boolean
useHdr
,
long
presentationTimeUs
)
{
float
[]
tempResultMatrix
=
new
float
[
16
];
float
[]
compositeRgbaMatrix
=
new
float
[
16
];
Matrix
.
setIdentityM
(
compositeRgbaMatrix
,
/* smOffset= */
0
);
for
(
int
i
=
0
;
i
<
rgbMatrices
.
size
();
i
++)
{
Matrix
.
multiplyMM
(
/* result= */
tempResultMatrix
,
/* resultOffset= */
0
,
/* lhs= */
rgbMatrices
.
get
(
i
).
getMatrix
(
presentationTimeUs
,
useHdr
),
/* lhsOffset= */
0
,
/* rhs= */
compositeRgbaMatrix
,
/* rhsOffset= */
0
);
System
.
arraycopy
(
/* src= */
tempResultMatrix
,
/* srcPos= */
0
,
/* dest= */
compositeRgbaMatrix
,
/* destPost= */
0
,
/* length= */
tempResultMatrix
.
length
);
}
return
compositeRgbaMatrix
;
}
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java
View file @
672405af
...
@@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState;
...
@@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import
android.content.Context
;
import
android.content.Context
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.util.UnstableApi
;
import
androidx.media3.common.util.UnstableApi
;
import
com.google.common.collect.ImmutableList
;
import
org.checkerframework.checker.nullness.qual.MonotonicNonNull
;
import
org.checkerframework.checker.nullness.qual.MonotonicNonNull
;
/** Provides common color filters. */
/** Provides common color filters. */
...
@@ -91,9 +92,13 @@ public class RgbFilter implements RgbMatrix {
...
@@ -91,9 +92,13 @@ public class RgbFilter implements RgbMatrix {
}
}
@Override
@Override
public
RgbMatrix
Processor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
public
MatrixTransformation
Processor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
checkForConsistentHdrSetting
(
useHdr
);
checkForConsistentHdrSetting
(
useHdr
);
return
new
RgbMatrixProcessor
(
context
,
/* rgbMatrix= */
this
,
useHdr
);
return
MatrixTransformationProcessor
.
create
(
context
,
/* matrixTransformations= */
ImmutableList
.
of
(),
/* rgbMatrices= */
ImmutableList
.
of
(
this
),
useHdr
);
}
}
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrix.java
View file @
672405af
...
@@ -19,6 +19,7 @@ package androidx.media3.effect;
...
@@ -19,6 +19,7 @@ package androidx.media3.effect;
import
android.content.Context
;
import
android.content.Context
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.util.UnstableApi
;
import
androidx.media3.common.util.UnstableApi
;
import
com.google.common.collect.ImmutableList
;
/**
/**
* Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
* Specifies a 4x4 RGB color transformation matrix to apply to each frame in the fragment shader.
...
@@ -39,8 +40,12 @@ public interface RgbMatrix extends GlEffect {
...
@@ -39,8 +40,12 @@ public interface RgbMatrix extends GlEffect {
float
[]
getMatrix
(
long
presentationTimeUs
,
boolean
useHdr
);
float
[]
getMatrix
(
long
presentationTimeUs
,
boolean
useHdr
);
@Override
@Override
default
RgbMatrix
Processor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
default
MatrixTransformation
Processor
toGlTextureProcessor
(
Context
context
,
boolean
useHdr
)
throws
FrameProcessingException
{
throws
FrameProcessingException
{
return
new
RgbMatrixProcessor
(
context
,
/* rgbMatrix= */
this
,
useHdr
);
return
MatrixTransformationProcessor
.
create
(
context
,
/* matrixTransformations= */
ImmutableList
.
of
(),
/* rgbMatrices= */
ImmutableList
.
of
(
this
),
useHdr
);
}
}
}
}
libraries/effect/src/main/java/androidx/media3/effect/RgbMatrixProcessor.java
deleted
100644 → 0
View file @
3a2e0d37
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
androidx
.
media3
.
effect
;
import
android.content.Context
;
import
android.opengl.GLES20
;
import
android.opengl.Matrix
;
import
android.util.Pair
;
import
androidx.media3.common.FrameProcessingException
;
import
androidx.media3.common.util.GlProgram
;
import
androidx.media3.common.util.GlUtil
;
import
com.google.common.collect.ImmutableList
;
import
java.io.IOException
;
/**
* Applies a sequence of {@link RgbMatrix} to each frame.
*
* <p>After applying all {@link RgbMatrix} instances, color values are clamped to the limits of the
* color space. Intermediate results are not clamped.
*/
/* package */
final
class
RgbMatrixProcessor
extends
SingleFrameGlTextureProcessor
{
private
static
final
String
VERTEX_SHADER_PATH
=
"shaders/vertex_shader_transformation_es2.glsl"
;
private
static
final
String
FRAGMENT_SHADER_PATH
=
"shaders/fragment_shader_transformation_es2.glsl"
;
private
final
GlProgram
glProgram
;
private
final
ImmutableList
<
RgbMatrix
>
rgbMatrices
;
private
final
boolean
useHdr
;
// TODO(b/239757183): Merge RgbMatrixProcessor with MatrixTransformationProcessor.
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrix The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public
RgbMatrixProcessor
(
Context
context
,
RgbMatrix
rgbMatrix
,
boolean
useHdr
)
throws
FrameProcessingException
{
this
(
context
,
ImmutableList
.
of
(
rgbMatrix
),
useHdr
);
}
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param rgbMatrices The {@link RgbMatrix} to apply to each frame.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public
RgbMatrixProcessor
(
Context
context
,
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
boolean
useHdr
)
throws
FrameProcessingException
{
super
(
useHdr
);
this
.
rgbMatrices
=
rgbMatrices
;
this
.
useHdr
=
useHdr
;
try
{
glProgram
=
new
GlProgram
(
context
,
VERTEX_SHADER_PATH
,
FRAGMENT_SHADER_PATH
);
}
catch
(
IOException
|
GlUtil
.
GlException
e
)
{
throw
new
FrameProcessingException
(
e
);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram
.
setBufferAttribute
(
"aFramePosition"
,
GlUtil
.
getNormalizedCoordinateBounds
(),
GlUtil
.
HOMOGENEOUS_COORDINATE_VECTOR_SIZE
);
float
[]
identityMatrix
=
new
float
[
16
];
Matrix
.
setIdentityM
(
identityMatrix
,
/* smOffset= */
0
);
glProgram
.
setFloatsUniform
(
"uTransformationMatrix"
,
identityMatrix
);
glProgram
.
setFloatsUniform
(
"uTexTransformationMatrix"
,
identityMatrix
);
}
@Override
public
Pair
<
Integer
,
Integer
>
configure
(
int
inputWidth
,
int
inputHeight
)
{
return
Pair
.
create
(
inputWidth
,
inputHeight
);
}
private
static
float
[]
createCompositeRgbaMatrixArray
(
ImmutableList
<
RgbMatrix
>
rgbMatrices
,
boolean
useHdr
,
long
presentationTimeUs
)
{
float
[]
tempResultMatrix
=
new
float
[
16
];
float
[]
compositeRgbaMatrix
=
new
float
[
16
];
Matrix
.
setIdentityM
(
compositeRgbaMatrix
,
/* smOffset= */
0
);
for
(
int
i
=
0
;
i
<
rgbMatrices
.
size
();
i
++)
{
Matrix
.
multiplyMM
(
/* result= */
tempResultMatrix
,
/* resultOffset= */
0
,
/* lhs= */
rgbMatrices
.
get
(
i
).
getMatrix
(
presentationTimeUs
,
useHdr
),
/* lhsOffset= */
0
,
/* rhs= */
compositeRgbaMatrix
,
/* rhsOffset= */
0
);
System
.
arraycopy
(
/* src= */
tempResultMatrix
,
/* srcPos= */
0
,
/* dest= */
compositeRgbaMatrix
,
/* destPost= */
0
,
/* length= */
tempResultMatrix
.
length
);
}
return
compositeRgbaMatrix
;
}
@Override
public
void
drawFrame
(
int
inputTexId
,
long
presentationTimeUs
)
throws
FrameProcessingException
{
// TODO(b/239431666): Add caching for compacting Matrices.
float
[]
rgbMatrixArray
=
createCompositeRgbaMatrixArray
(
rgbMatrices
,
useHdr
,
presentationTimeUs
);
try
{
glProgram
.
use
();
glProgram
.
setSamplerTexIdUniform
(
"uTexSampler"
,
inputTexId
,
/* texUnitIndex= */
0
);
glProgram
.
setFloatsUniform
(
"uColorMatrix"
,
rgbMatrixArray
);
glProgram
.
bindAttributesAndUniforms
();
// The four-vertex triangle strip forms a quad.
GLES20
.
glDrawArrays
(
GLES20
.
GL_TRIANGLE_STRIP
,
/* first= */
0
,
/* count= */
4
);
}
catch
(
GlUtil
.
GlException
e
)
{
throw
new
FrameProcessingException
(
e
,
presentationTimeUs
);
}
}
@Override
public
void
release
()
throws
FrameProcessingException
{
super
.
release
();
try
{
glProgram
.
delete
();
}
catch
(
GlUtil
.
GlException
e
)
{
throw
new
FrameProcessingException
(
e
);
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment