Skip to content

SDK / exoplayer

Go to a project
Commit f67c1a73 by huangdarwin Committed by Rohit Singh
Options

HDR: Use FP16 color representation for texture processors. 

* Introduced `useHdr` for `GlEffect#toGlTextureProcessor`, so
  `TextureProcessor` implementations can decide how to handle HDR.
* Creating FP16 color textures for HDR input.

Tested via manual testing, adding a no-op GlEffectWrapper to the transformation to
force use of intermediate textures, adding a linear ramp to the fragment shader,
and trying to ascertain that there's a real reduction in posterization when
switching from 4-bit to 8-bit unsigned bytes, and again from 8-bit unsigned bytes
to 16-bit floating point.

PiperOrigin-RevId: 461613117
parent fd046bd2
Inline Side-by-side
Showing with 149 additions and 45 deletions
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
*/ */
package com.google.android.exoplayer2.transformerdemo; package com.google.android.exoplayer2.transformerdemo;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
...@@ -64,9 +65,14 @@ import java.util.Locale; ...@@ -64,9 +65,14 @@ import java.util.Locale;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public BitmapOverlayProcessor(Context context) throws FrameProcessingException { public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "BitmapOverlayProcessor does not support HDR colors.");
paint = new Paint(); paint = new Paint();
paint.setTextSize(64); paint.setTextSize(64);
paint.setAntiAlias(true); paint.setAntiAlias(true);
...@@ -85,7 +91,11 @@ import java.util.Locale; ...@@ -85,7 +91,11 @@ import java.util.Locale;
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
try { try {
bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT); bitmapTexId =
GlUtil.createTexture(
BITMAP_WIDTH_HEIGHT,
BITMAP_WIDTH_HEIGHT,
/* useHighPrecisionColorComponents= */ false);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0); GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
......
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java
...@@ -52,6 +52,8 @@ import java.io.IOException; ...@@ -52,6 +52,8 @@ import java.io.IOException;
* <p>The parameters are given in normalized texture coordinates from 0 to 1. * <p>The parameters are given in normalized texture coordinates from 0 to 1.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param centerX The x-coordinate of the center of the effect. * @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect. * @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect. * @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
...@@ -61,12 +63,15 @@ import java.io.IOException; ...@@ -61,12 +63,15 @@ import java.io.IOException;
*/ */
public PeriodicVignetteProcessor( public PeriodicVignetteProcessor(
Context context, Context context,
boolean useHdr,
float centerX, float centerX,
float centerY, float centerY,
float minInnerRadius, float minInnerRadius,
float maxInnerRadius, float maxInnerRadius,
float outerRadius) float outerRadius)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "PeriodicVignetteProcessor does not support HDR color spaces.");
checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius); checkArgument(maxInnerRadius <= outerRadius);
this.minInnerRadius = minInnerRadius; this.minInnerRadius = minInnerRadius;
......
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java
...@@ -277,13 +277,15 @@ public final class TransformerActivity extends AppCompatActivity { ...@@ -277,13 +277,15 @@ public final class TransformerActivity extends AppCompatActivity {
Class<?> clazz = Class<?> clazz =
Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor"); Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor");
Constructor<?> constructor = Constructor<?> constructor =
clazz.getConstructor(Context.class, String.class, String.class, String.class); clazz.getConstructor(
Context.class, Boolean.class, String.class, String.class, String.class);
effects.add( effects.add(
(Context context) -> { (Context context, boolean useHdr) -> {
try { try {
return (GlTextureProcessor) return (GlTextureProcessor)
constructor.newInstance( constructor.newInstance(
context, context,
useHdr,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video", /* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video"); /* outputStreamName= */ "output_video");
...@@ -298,9 +300,10 @@ public final class TransformerActivity extends AppCompatActivity { ...@@ -298,9 +300,10 @@ public final class TransformerActivity extends AppCompatActivity {
} }
if (selectedEffects[2]) { if (selectedEffects[2]) {
effects.add( effects.add(
(Context context) -> (Context context, boolean useHdr) ->
new PeriodicVignetteProcessor( new PeriodicVignetteProcessor(
context, context,
useHdr,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat( /* minInnerRadius= */ bundle.getFloat(
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
*/ */
package com.google.android.exoplayer2.transformerdemo; package com.google.android.exoplayer2.transformerdemo;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
...@@ -70,14 +71,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -70,14 +71,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param graphName Name of a MediaPipe graph asset to load. * @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph. * @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph.
*/ */
@SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23. @SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23.
public MediaPipeProcessor( public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName) { Context context,
boolean useHdr,
String graphName,
String inputStreamName,
String outputStreamName) {
checkState(LOADER.isAvailable()); checkState(LOADER.isAvailable());
// TODO(b/227624622): Confirm whether MediaPipeProcessor could support HDR colors.
checkArgument(!useHdr, "MediaPipeProcessor does not support HDR colors.");
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(
......
library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package com.google.android.exoplayer2.util; package com.google.android.exoplayer2.util;
import static android.opengl.GLU.gluErrorString; import static android.opengl.GLU.gluErrorString;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.content.pm.PackageManager; import android.content.pm.PackageManager;
...@@ -26,6 +27,7 @@ import android.opengl.EGLDisplay; ...@@ -26,6 +27,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES11Ext; import android.opengl.GLES11Ext;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.GLES30;
import androidx.annotation.DoNotInline; import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
...@@ -488,12 +490,37 @@ public final class GlUtil { ...@@ -488,12 +490,37 @@ public final class GlUtil {
} }
/** /**
* Returns the texture identifier for a newly-allocated texture with the specified dimensions. * Allocates a new RGBA texture with the specified dimensions and color component precision.
* *
* @param width of the new texture in pixels * @param width The width of the new texture in pixels.
* @param height of the new texture in pixels * @param height The height of the new texture in pixels.
* @param useHighPrecisionColorComponents If {@code false}, uses 8-bit unsigned bytes. If {@code
* true}, use 16-bit (half-precision) floating-point.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
*/ */
public static int createTexture(int width, int height) throws GlException { public static int createTexture(int width, int height, boolean useHighPrecisionColorComponents)
throws GlException {
// TODO(227624622): Implement a pixel test that confirms 16f has less posterization.
if (useHighPrecisionColorComponents) {
checkState(Util.SDK_INT >= 18, "GLES30 extensions are not supported below API 18.");
return createTexture(width, height, GLES30.GL_RGBA16F, GLES30.GL_HALF_FLOAT);
}
return createTexture(width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE);
}
/**
* Allocates a new RGBA texture with the specified dimensions and color component precision.
*
* @param width The width of the new texture in pixels.
* @param height The height of the new texture in pixels.
* @param internalFormat The number of color components in the texture, as well as their format.
* @param type The data type of the pixel data.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
*/
private static int createTexture(int width, int height, int internalFormat, int type)
throws GlException {
assertValidTextureSize(width, height); assertValidTextureSize(width, height);
int texId = generateTexture(); int texId = generateTexture();
bindTexture(GLES20.GL_TEXTURE_2D, texId); bindTexture(GLES20.GL_TEXTURE_2D, texId);
...@@ -501,12 +528,12 @@ public final class GlUtil { ...@@ -501,12 +528,12 @@ public final class GlUtil {
GLES20.glTexImage2D( GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_2D,
/* level= */ 0, /* level= */ 0,
GLES20.GL_RGBA, internalFormat,
width, width,
height, height,
/* border= */ 0, /* border= */ 0,
GLES20.GL_RGBA, GLES20.GL_RGBA,
GLES20.GL_UNSIGNED_BYTE, type,
byteBuffer); byteBuffer);
checkGlError(); checkGlError();
return texId; return texId;
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java
...@@ -189,6 +189,7 @@ public class BitmapTestUtil { ...@@ -189,6 +189,7 @@ public class BitmapTestUtil {
public static Bitmap createArgb8888BitmapFromCurrentGlFramebuffer(int width, int height) public static Bitmap createArgb8888BitmapFromCurrentGlFramebuffer(int width, int height)
throws GlUtil.GlException { throws GlUtil.GlException {
ByteBuffer rgba8888Buffer = ByteBuffer.allocateDirect(width * height * 4); ByteBuffer rgba8888Buffer = ByteBuffer.allocateDirect(width * height * 4);
// TODO(b/227624622): Add support for reading HDR bitmaps.
GLES20.glReadPixels( GLES20.glReadPixels(
0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgba8888Buffer); 0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgba8888Buffer);
GlUtil.checkGlError(); GlUtil.checkGlError();
...@@ -208,7 +209,10 @@ public class BitmapTestUtil { ...@@ -208,7 +209,10 @@ public class BitmapTestUtil {
* @return The identifier of the newly created texture. * @return The identifier of the newly created texture.
*/ */
public static int createGlTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException { public static int createGlTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException {
int texId = GlUtil.createTexture(bitmap.getWidth(), bitmap.getHeight()); // TODO(b/227624622): Add support for reading HDR bitmaps.
int texId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
// Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down
// while OpenGL's positive y-axis points up. // while OpenGL's positive y-axis points up.
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0); GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0);
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/CropPixelTest.java
...@@ -89,7 +89,7 @@ public final class CropPixelTest { ...@@ -89,7 +89,7 @@ public final class CropPixelTest {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1) new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
...@@ -113,7 +113,7 @@ public final class CropPixelTest { ...@@ -113,7 +113,7 @@ public final class CropPixelTest {
String testId = "drawFrame_cropSmaller"; String testId = "drawFrame_cropSmaller";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f) new Crop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH);
...@@ -137,7 +137,7 @@ public final class CropPixelTest { ...@@ -137,7 +137,7 @@ public final class CropPixelTest {
String testId = "drawFrame_cropLarger"; String testId = "drawFrame_cropLarger";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f) new Crop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH);
...@@ -157,7 +157,9 @@ public final class CropPixelTest { ...@@ -157,7 +157,9 @@ public final class CropPixelTest {
} }
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
outputTexId = GlUtil.createTexture(outputWidth, outputHeight); outputTexId =
GlUtil.createTexture(
outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
checkNotNull(eglDisplay), checkNotNull(eglDisplay),
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java
...@@ -320,6 +320,9 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -320,6 +320,9 @@ public final class GlEffectsFrameProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
// TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps, using
// GlEffectWrapper to ensure usage of intermediate textures.
/** /**
* Set up and prepare the first frame from an input video, as well as relevant test * Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link GlEffectsFrameProcessor}, and output * infrastructure. The frame will be sent towards the {@link GlEffectsFrameProcessor}, and output
...@@ -379,7 +382,7 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -379,7 +382,7 @@ public final class GlEffectsFrameProcessorPixelTest {
/* streamOffsetUs= */ 0L, /* streamOffsetUs= */ 0L,
effects, effects,
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* enableExperimentalHdrEditing= */ false)); /* useHdr= */ false));
glEffectsFrameProcessor.setInputFrameInfo( glEffectsFrameProcessor.setInputFrameInfo(
new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio)); new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio));
glEffectsFrameProcessor.registerInputFrame(); glEffectsFrameProcessor.registerInputFrame();
...@@ -494,9 +497,9 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -494,9 +497,9 @@ public final class GlEffectsFrameProcessorPixelTest {
} }
@Override @Override
public GlTextureProcessor toGlTextureProcessor(Context context) public GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
return effect.toGlTextureProcessor(context); return effect.toGlTextureProcessor(context, useHdr);
} }
} }
} }
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessorPixelTest.java
...@@ -72,7 +72,7 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -72,7 +72,7 @@ public final class MatrixTransformationProcessorPixelTest {
EGLSurface placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); EGLSurface placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay);
GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, width, height); GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, width, height);
inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap);
outputTexId = GlUtil.createTexture(width, height); outputTexId = GlUtil.createTexture(width, height, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglContext, placeholderEglSurface, frameBuffer, width, height); eglDisplay, eglContext, placeholderEglSurface, frameBuffer, width, height);
...@@ -93,7 +93,10 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -93,7 +93,10 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
Matrix identityMatrix = new Matrix(); Matrix identityMatrix = new Matrix();
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(context, (long presentationTimeUs) -> identityMatrix); new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> identityMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
...@@ -117,7 +120,9 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -117,7 +120,9 @@ public final class MatrixTransformationProcessorPixelTest {
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
...@@ -141,7 +146,9 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -141,7 +146,9 @@ public final class MatrixTransformationProcessorPixelTest {
scaleNarrowMatrix.postScale(.5f, 1.2f); scaleNarrowMatrix.postScale(.5f, 1.2f);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
...@@ -165,7 +172,9 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -165,7 +172,9 @@ public final class MatrixTransformationProcessorPixelTest {
rotate90Matrix.postRotate(/* degrees= */ 90); rotate90Matrix.postRotate(/* degrees= */ 90);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
...@@ -181,4 +190,6 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -181,4 +190,6 @@ public final class MatrixTransformationProcessorPixelTest {
expectedBitmap, actualBitmap, testId); expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
// TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps.
} }
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/PresentationPixelTest.java
...@@ -97,7 +97,8 @@ public final class PresentationPixelTest { ...@@ -97,7 +97,8 @@ public final class PresentationPixelTest {
public void drawFrame_noEdits_producesExpectedOutput() throws Exception { public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForHeight(C.LENGTH_UNSET).toGlTextureProcessor(context); Presentation.createForHeight(C.LENGTH_UNSET)
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
...@@ -122,7 +123,7 @@ public final class PresentationPixelTest { ...@@ -122,7 +123,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_scaleToFit_narrow"; String testId = "drawFrame_changeAspectRatio_scaleToFit_narrow";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -148,7 +149,7 @@ public final class PresentationPixelTest { ...@@ -148,7 +149,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_scaleToFit_wide"; String testId = "drawFrame_changeAspectRatio_scaleToFit_wide";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -175,7 +176,7 @@ public final class PresentationPixelTest { ...@@ -175,7 +176,7 @@ public final class PresentationPixelTest {
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio( Presentation.createForAspectRatio(
/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) /* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -202,7 +203,7 @@ public final class PresentationPixelTest { ...@@ -202,7 +203,7 @@ public final class PresentationPixelTest {
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio( Presentation.createForAspectRatio(
/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) /* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -228,7 +229,7 @@ public final class PresentationPixelTest { ...@@ -228,7 +229,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_stretchToFit_narrow"; String testId = "drawFrame_changeAspectRatio_stretchToFit_narrow";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_STRETCH_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_STRETCH_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -254,7 +255,7 @@ public final class PresentationPixelTest { ...@@ -254,7 +255,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_stretchToFit_wide"; String testId = "drawFrame_changeAspectRatio_stretchToFit_wide";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_STRETCH_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_STRETCH_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
...@@ -275,7 +276,9 @@ public final class PresentationPixelTest { ...@@ -275,7 +276,9 @@ public final class PresentationPixelTest {
} }
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
outputTexId = GlUtil.createTexture(outputWidth, outputHeight); outputTexId =
GlUtil.createTexture(
outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
checkNotNull(eglDisplay), checkNotNull(eglDisplay),
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
...@@ -82,6 +82,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -82,6 +82,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable @Nullable
private EGLSurface outputEglSurface; private EGLSurface outputEglSurface;
// TODO(b/227624622): Instead of inputting useHdr, input ColorInfo to handle HLG and PQ
// differently.
public FinalMatrixTransformationProcessorWrapper( public FinalMatrixTransformationProcessorWrapper(
Context context, Context context,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java
...@@ -21,11 +21,18 @@ import android.content.Context; ...@@ -21,11 +21,18 @@ import android.content.Context;
* Interface for a video frame effect with a {@link GlTextureProcessor} implementation. * Interface for a video frame effect with a {@link GlTextureProcessor} implementation.
* *
* <p>Implementations contain information specifying the effect and can be {@linkplain * <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the * #toGlTextureProcessor(Context, boolean) converted} to a {@link GlTextureProcessor} which applies
* effect. * the effect.
*/ */
public interface GlEffect { public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ /**
GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException; * Returns a {@link SingleFrameGlTextureProcessor} that applies the effect.
*
* @param context A {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException;
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java
...@@ -189,7 +189,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -189,7 +189,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false; sampleFromExternalTexture = false;
} }
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); textureProcessorListBuilder.add(effect.toGlTextureProcessor(context, useHdr));
} }
textureProcessorListBuilder.add( textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper( new FinalMatrixTransformationProcessorWrapper(
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java
...@@ -49,8 +49,8 @@ public interface GlMatrixTransformation extends GlEffect { ...@@ -49,8 +49,8 @@ public interface GlMatrixTransformation extends GlEffect {
float[] getGlMatrixArray(long presentationTimeUs); float[] getGlMatrixArray(long presentationTimeUs);
@Override @Override
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
return new MatrixTransformationProcessor(context, this); return new MatrixTransformationProcessor(context, useHdr, /* matrixTransformation= */ this);
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java
...@@ -97,34 +97,40 @@ import java.util.Arrays; ...@@ -97,34 +97,40 @@ import java.util.Arrays;
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) public MatrixTransformationProcessor(
Context context, boolean useHdr, MatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
context, context,
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false, /* sampleFromExternalTexture= */ false,
/* useHdr= */ false); useHdr);
} }
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) public MatrixTransformationProcessor(
Context context, boolean useHdr, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
context, context,
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false, /* sampleFromExternalTexture= */ false,
/* useHdr= */ false); useHdr);
} }
/** /**
...@@ -147,6 +153,7 @@ import java.util.Arrays; ...@@ -147,6 +153,7 @@ import java.util.Arrays;
boolean sampleFromExternalTexture, boolean sampleFromExternalTexture,
boolean useHdr) boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr);
if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) { if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException( throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing."); "The EXT_YUV_target extension is required for HDR editing.");
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java
...@@ -38,6 +38,17 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso ...@@ -38,6 +38,17 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
private int inputHeight; private int inputHeight;
private @MonotonicNonNull TextureInfo outputTexture; private @MonotonicNonNull TextureInfo outputTexture;
private boolean outputTextureInUse; private boolean outputTextureInUse;
private final boolean useHdr;
/**
* Creates a {@code SingleFrameGlTextureProcessor} instance.
*
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
public SingleFrameGlTextureProcessor(boolean useHdr) {
this.useHdr = useHdr;
}
/** /**
* Configures the texture processor based on the input dimensions. * Configures the texture processor based on the input dimensions.
...@@ -116,7 +127,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso ...@@ -116,7 +127,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
if (outputTexture != null) { if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId); GlUtil.deleteTexture(outputTexture.texId);
} }
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
int outputFboId = GlUtil.createFboForTexture(outputTexId); int outputFboId = GlUtil.createFboForTexture(outputTexId);
outputTexture = outputTexture =
new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight()); new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment