Skip to content

SDK / exoplayer

Go to a project
Commit e444eaa5 by huangdarwin Committed by Marc Baechinger
Options

HDR: Input ColorInfo to the FrameProcessor. 

This allows the GlEffectsFrameProcessor to later handle HLG and PQ
differently, or limited and full color range differently.

No functional change intended in this CL.

PiperOrigin-RevId: 466070764
parent 3aa99d5e
Inline Side-by-side
Showing with 153 additions and 77 deletions
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
...@@ -42,6 +42,7 @@ import androidx.media3.common.FrameProcessor; ...@@ -42,6 +42,7 @@ import androidx.media3.common.FrameProcessor;
import androidx.media3.common.SurfaceInfo; import androidx.media3.common.SurfaceInfo;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List; import java.util.List;
...@@ -388,7 +389,7 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -388,7 +389,7 @@ public final class GlEffectsFrameProcessorPixelTest {
}, },
effects, effects,
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* useHdr= */ false)); ColorInfo.SDR_BT709_LIMITED));
glEffectsFrameProcessor.setInputFrameInfo( glEffectsFrameProcessor.setInputFrameInfo(
new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0)); new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0));
glEffectsFrameProcessor.registerInputFrame(); glEffectsFrameProcessor.registerInputFrame();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
...@@ -40,6 +40,7 @@ import com.google.android.exoplayer2.C; ...@@ -40,6 +40,7 @@ import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.Queue; import java.util.Queue;
...@@ -69,7 +70,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -69,7 +70,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener; private final FrameProcessor.Listener frameProcessorListener;
private final boolean sampleFromExternalTexture; private final boolean sampleFromExternalTexture;
private final boolean useHdr; private final ColorInfo colorInfo;
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
private final Queue<Long> streamOffsetUsQueue; private final Queue<Long> streamOffsetUsQueue;
...@@ -91,8 +92,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -91,8 +92,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable @Nullable
private EGLSurface outputEglSurface; private EGLSurface outputEglSurface;
// TODO(b/227624622): Instead of inputting useHdr, input ColorInfo to handle HLG and PQ
// differently.
public FinalMatrixTransformationProcessorWrapper( public FinalMatrixTransformationProcessorWrapper(
Context context, Context context,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
...@@ -101,7 +100,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -101,7 +100,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessor.Listener frameProcessorListener, FrameProcessor.Listener frameProcessorListener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture, boolean sampleFromExternalTexture,
boolean useHdr) { ColorInfo colorInfo) {
this.context = context; this.context = context;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
...@@ -109,7 +108,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -109,7 +108,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener; this.frameProcessorListener = frameProcessorListener;
this.sampleFromExternalTexture = sampleFromExternalTexture; this.sampleFromExternalTexture = sampleFromExternalTexture;
this.useHdr = useHdr; this.colorInfo = colorInfo;
textureTransformMatrix = new float[16]; textureTransformMatrix = new float[16];
Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0); Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
...@@ -215,7 +214,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -215,7 +214,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface; @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) { if (outputEglSurface == null) {
if (useHdr) { boolean colorInfoIsHdr = ColorInfo.isHdr(colorInfo);
if (colorInfoIsHdr) {
outputEglSurface = GlUtil.getEglSurfaceRgba1010102(eglDisplay, outputSurfaceInfo.surface); outputEglSurface = GlUtil.getEglSurfaceRgba1010102(eglDisplay, outputSurfaceInfo.surface);
} else { } else {
outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
...@@ -227,7 +227,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -227,7 +227,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width, outputSurfaceInfo.height); outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) { if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper = debugSurfaceViewWrapper =
new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView); new SurfaceViewWrapper(eglDisplay, eglContext, colorInfoIsHdr, debugSurfaceView);
} }
this.debugSurfaceView = debugSurfaceView; this.debugSurfaceView = debugSurfaceView;
} }
...@@ -266,7 +266,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -266,7 +266,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
context, context,
matrixTransformationListBuilder.build(), matrixTransformationListBuilder.build(),
sampleFromExternalTexture, sampleFromExternalTexture,
useHdr, colorInfo,
/* outputOpticalColors= */ true); /* outputOpticalColors= */ true);
matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
Pair<Integer, Integer> outputSize = Pair<Integer, Integer> outputSize =
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
...@@ -37,6 +37,7 @@ import androidx.media3.common.SurfaceInfo; ...@@ -37,6 +37,7 @@ import androidx.media3.common.SurfaceInfo;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
...@@ -66,7 +67,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -66,7 +67,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean useHdr) ColorInfo colorInfo)
throws FrameProcessingException { throws FrameProcessingException {
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
...@@ -79,7 +80,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -79,7 +80,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
listener, listener,
effects, effects,
debugViewProvider, debugViewProvider,
useHdr, colorInfo,
singleThreadExecutorService)); singleThreadExecutorService));
try { try {
...@@ -109,11 +110,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -109,11 +110,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean useHdr, ColorInfo colorInfo,
ExecutorService singleThreadExecutorService) ExecutorService singleThreadExecutorService)
throws GlUtil.GlException, FrameProcessingException { throws GlUtil.GlException, FrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
// TODO(b/237674316): Delay initialization of things requiring the colorInfo, to
// configure based on the color info from the decoder output media format instead.
boolean useHdr = ColorInfo.isHdr(colorInfo);
EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext = EGLContext eglContext =
useHdr useHdr
...@@ -131,7 +135,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -131,7 +135,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList<GlTextureProcessor> textureProcessors = ImmutableList<GlTextureProcessor> textureProcessors =
getGlTextureProcessorsForGlEffects( getGlTextureProcessorsForGlEffects(
context, effects, eglDisplay, eglContext, listener, debugViewProvider, useHdr); context, effects, eglDisplay, eglContext, listener, debugViewProvider, colorInfo);
FrameProcessingTaskExecutor frameProcessingTaskExecutor = FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener); chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
...@@ -162,7 +166,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -162,7 +166,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
EGLContext eglContext, EGLContext eglContext,
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean useHdr) ColorInfo colorInfo)
throws FrameProcessingException { throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder = ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
...@@ -185,12 +189,13 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -185,12 +189,13 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
context, context,
matrixTransformations, matrixTransformations,
sampleFromExternalTexture, sampleFromExternalTexture,
useHdr, colorInfo,
/* outputOpticalColors= */ false)); /* outputOpticalColors= */ false));
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false; sampleFromExternalTexture = false;
} }
textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, useHdr)); textureProcessorListBuilder.add(
glEffect.toGlTextureProcessor(context, ColorInfo.isHdr(colorInfo)));
} }
textureProcessorListBuilder.add( textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper( new FinalMatrixTransformationProcessorWrapper(
...@@ -201,7 +206,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ...@@ -201,7 +206,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
listener, listener,
debugViewProvider, debugViewProvider,
sampleFromExternalTexture, sampleFromExternalTexture,
useHdr)); colorInfo));
return textureProcessorListBuilder.build(); return textureProcessorListBuilder.build();
} }
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
...@@ -24,6 +24,7 @@ import android.util.Pair; ...@@ -24,6 +24,7 @@ import android.util.Pair;
import androidx.media3.common.FrameProcessingException; import androidx.media3.common.FrameProcessingException;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
...@@ -110,11 +111,13 @@ import java.util.Arrays; ...@@ -110,11 +111,13 @@ import java.util.Arrays;
Context context, boolean useHdr, MatrixTransformation matrixTransformation) Context context, boolean useHdr, MatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
createGlProgram(
context, context,
ImmutableList.of(matrixTransformation), /* inputOpticalColorsFromExternalTexture= */ false,
/* sampleFromExternalTexture= */ false,
useHdr, useHdr,
/* outputOpticalColors= */ false); /* outputOpticalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
} }
/** /**
...@@ -131,43 +134,89 @@ import java.util.Arrays; ...@@ -131,43 +134,89 @@ import java.util.Arrays;
Context context, boolean useHdr, GlMatrixTransformation matrixTransformation) Context context, boolean useHdr, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
createGlProgram(
context, context,
ImmutableList.of(matrixTransformation), /* inputOpticalColorsFromExternalTexture= */ false,
/* sampleFromExternalTexture= */ false,
useHdr, useHdr,
/* outputOpticalColors= */ false); /* outputOpticalColors= */ false),
ImmutableList.of(matrixTransformation),
useHdr);
} }
/** /**
* Creates a new instance. * Creates a new instance.
* *
* <p>Able to convert optical {@link ColorInfo} inputs and outputs to and from the intermediate
* {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma RGB BT.709 for SDR.
*
* @param context The {@link Context}. * @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. * apply to each frame in order.
* @param sampleFromExternalTexture Whether the input will be provided using an external texture. * @param inputOpticalColorsFromExternalTexture Whether optical color input will be provided using
* If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to * an external texture. If {@code true}, the caller should use {@link
* provide the transformation matrix associated with the external texture. * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code * the external texture.
* EXT_YUV_target} OpenGL extension. * @param opticalColorInfo The optical {@link ColorInfo}, only used to transform between color
* @param outputOpticalColors If {@code true} and {@code useHdr} is also {@code true}, outputs a * spaces and transfers, when {@code inputOpticalColorsFromExternalTexture} or {@code
* non-linear optical, or display light colors, possibly by applying the EOTF (Electro-optical * outputOpticalColors} are {@code true}. If it {@link ColorInfo#isHdr(ColorInfo)},
* transfer function). Otherwise, outputs linear electrical colors. * intermediate {@link GlTextureProcessor} colors will be in linear RGB BT.2020. Otherwise,
* these colors will be in gamma RGB BT.709.
* @param outputOpticalColors If {@code true}, outputs {@code opticalColorInfo}. If {@code false},
* outputs intermediate colors of linear RGB BT.2020 if {@code opticalColorInfo} {@link
* ColorInfo#isHdr(ColorInfo)}, and gamma RGB BT.709 otherwise.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported. * operation fails or is unsupported.
*/ */
public MatrixTransformationProcessor( public MatrixTransformationProcessor(
Context context, Context context,
ImmutableList<GlMatrixTransformation> matrixTransformations, ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean sampleFromExternalTexture, boolean inputOpticalColorsFromExternalTexture,
boolean useHdr, ColorInfo opticalColorInfo,
boolean outputOpticalColors) boolean outputOpticalColors)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr); this(
if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) { createGlProgram(
context,
inputOpticalColorsFromExternalTexture,
ColorInfo.isHdr(opticalColorInfo),
outputOpticalColors),
matrixTransformations,
ColorInfo.isHdr(opticalColorInfo));
if (!ColorInfo.isHdr(opticalColorInfo) || !inputOpticalColorsFromExternalTexture) {
return;
}
// TODO(b/227624622): Implement YUV to RGB conversions in COLOR_RANGE_LIMITED as well, using
// opticalColorInfo.colorRange to select between them.
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException( throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input."); "The EXT_YUV_target extension is required for HDR editing input.");
} }
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
// TODO(b/227624622): Implement PQ and gamma TFs, and use an @IntDef to select between HLG,
// PQ, and gamma, coming from opticalColorInfo.colorTransfer.
// Applying the OETF will output a linear signal. Not applying the OETF will output an optical
// signal.
glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
}
/**
* Creates a new instance.
*
* @param glProgram The {@link GlProgram}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
*/
private MatrixTransformationProcessor(
GlProgram glProgram,
ImmutableList<GlMatrixTransformation> matrixTransformations,
boolean useHdr) {
super(useHdr);
this.glProgram = glProgram;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
transformationMatrixCache = new float[matrixTransformations.size()][16]; transformationMatrixCache = new float[matrixTransformations.size()][16];
...@@ -175,42 +224,44 @@ import java.util.Arrays; ...@@ -175,42 +224,44 @@ import java.util.Arrays;
tempResultMatrix = new float[16]; tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE; visiblePolygon = NDC_SQUARE;
}
private static GlProgram createGlProgram(
Context context,
boolean inputOpticalColorsFromExternalTexture,
boolean useHdr,
boolean outputOpticalColors)
throws FrameProcessingException {
String vertexShaderFilePath; String vertexShaderFilePath;
String fragmentShaderFilePath; String fragmentShaderFilePath;
if (sampleFromExternalTexture) { if (inputOpticalColorsFromExternalTexture) {
vertexShaderFilePath = if (useHdr) {
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH;
useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; } else {
} else if (outputOpticalColors) { vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
vertexShaderFilePath = fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; }
fragmentShaderFilePath = } else if (outputOpticalColors && useHdr) {
useHdr ? FRAGMENT_SHADER_HLG_EOTF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_HLG_EOTF_ES3_PATH;
} else { } else {
vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH; vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH; fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
} }
GlProgram glProgram;
try { try {
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
} catch (IOException | GlUtil.GlException e) { } catch (IOException | GlUtil.GlException e) {
throw new FrameProcessingException(e); throw new FrameProcessingException(e);
} }
if (useHdr && sampleFromExternalTexture) {
// In HDR editing mode the decoder output is sampled in YUV.
glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
// TODO(b/227624622): Implement PQ, and use an @IntDef to select between HLG, PQ, and no
// transfer function.
// Applying the OETF will output a linear signal. Not applying the OETF will output an optical
// signal.
glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
}
float[] identityMatrix = new float[16]; float[] identityMatrix = new float[16];
Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0); Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
return glProgram;
} }
@Override @Override
...@@ -274,11 +325,11 @@ import java.util.Arrays; ...@@ -274,11 +325,11 @@ import java.util.Arrays;
visiblePolygon = NDC_SQUARE; visiblePolygon = NDC_SQUARE;
for (float[] transformationMatrix : transformationMatrixCache) { for (float[] transformationMatrix : transformationMatrixCache) {
Matrix.multiplyMM( Matrix.multiplyMM(
tempResultMatrix, /* result= */ tempResultMatrix,
/* resultOffset= */ 0, /* resultOffset= */ 0,
transformationMatrix, /* lhs= */ transformationMatrix,
/* lhsOffset= */ 0, /* lhsOffset= */ 0,
compositeTransformationMatrix, /* rhs= */ compositeTransformationMatrix,
/* rhsOffset= */ 0); /* rhsOffset= */ 0);
System.arraycopy( System.arraycopy(
/* src= */ tempResultMatrix, /* src= */ tempResultMatrix,
......
library/common/src/main/java/androidx/media3/common/FrameProcessor.java
...@@ -44,17 +44,17 @@ public interface FrameProcessor { ...@@ -44,17 +44,17 @@ public interface FrameProcessor {
* @param listener A {@link Listener}. * @param listener A {@link Listener}.
* @param effects The {@link Effect} instances to apply to each frame. * @param effects The {@link Effect} instances to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}. * @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal. * @param colorInfo The {@link ColorInfo} for input and output frames.
* @return A new instance. * @return A new instance.
* @throws FrameProcessingException If a problem occurs while creating the {@link * @throws FrameProcessingException If a problem occurs while creating the {@link
* FrameProcessor}. * FrameProcessor}.
*/ */
FrameProcessor create( FrameProcessor create(
Context context, Context context,
FrameProcessor.Listener listener, Listener listener,
List<Effect> effects, List<Effect> effects,
DebugViewProvider debugViewProvider, DebugViewProvider debugViewProvider,
boolean useHdr) ColorInfo colorInfo)
throws FrameProcessingException; throws FrameProcessingException;
} }
......
library/common/src/main/java/com/google/android/exoplayer2/video/ColorInfo.java
...@@ -30,7 +30,12 @@ import java.lang.annotation.Target; ...@@ -30,7 +30,12 @@ import java.lang.annotation.Target;
import java.util.Arrays; import java.util.Arrays;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
/** Stores color info. */ /**
* Stores color info.
*
* <p>When a {@code null} {@code ColorInfo} instance is used, this often represents a generic {@link
* #SDR_BT709_LIMITED} instance.
*/
public final class ColorInfo implements Bundleable { public final class ColorInfo implements Bundleable {
/** Color info representing SDR BT.709 limited range, which is a common SDR video color format. */ /** Color info representing SDR BT.709 limited range, which is a common SDR video color format. */
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -34,6 +34,7 @@ import androidx.media3.effect.ScaleToFitTransformation; ...@@ -34,6 +34,7 @@ import androidx.media3.effect.ScaleToFitTransformation;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo; import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
...@@ -141,10 +142,11 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -141,10 +142,11 @@ import org.checkerframework.dataflow.qual.Pure;
}, },
effectsListBuilder.build(), effectsListBuilder.build(),
debugViewProvider, debugViewProvider,
// HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
// implies that the OpenGL EXT_YUV_target extension is supported and hence the // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
/* useHdr= */ encoderWrapper.isHdrEditingEnabled()); // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
encoderWrapper.getSupportedInputColor());
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException( throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED); e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);
...@@ -154,7 +156,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -154,7 +156,8 @@ import org.checkerframework.dataflow.qual.Pure;
decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs)); decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
boolean isToneMappingRequired = boolean isToneMappingRequired =
ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled(); ColorInfo.isHdr(inputFormat.colorInfo)
&& !ColorInfo.isHdr(encoderWrapper.getSupportedInputColor());
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(
inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired); inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
...@@ -317,6 +320,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -317,6 +320,7 @@ import org.checkerframework.dataflow.qual.Pure;
*/ */
@VisibleForTesting @VisibleForTesting
/* package */ static final class EncoderWrapper { /* package */ static final class EncoderWrapper {
private static final String TAG = "EncoderWrapper";
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Format inputFormat; private final Format inputFormat;
...@@ -353,11 +357,24 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -353,11 +357,24 @@ import org.checkerframework.dataflow.qual.Pure;
requestedOutputMimeType, inputFormat.colorInfo); requestedOutputMimeType, inputFormat.colorInfo);
} }
/** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */ /** Returns the {@link ColorInfo} expected from the input surface. */
public boolean isHdrEditingEnabled() { public ColorInfo getSupportedInputColor() {
return transformationRequest.enableHdrEditing boolean isHdrEditingEnabled =
transformationRequest.enableHdrEditing
&& !transformationRequest.enableRequestSdrToneMapping && !transformationRequest.enableRequestSdrToneMapping
&& !supportedEncoderNamesForHdrEditing.isEmpty(); && !supportedEncoderNamesForHdrEditing.isEmpty();
boolean isInputToneMapped = !isHdrEditingEnabled && ColorInfo.isHdr(inputFormat.colorInfo);
if (isInputToneMapped) {
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
// TODO(b/237674316): Get the color info from the decoder output media format instead.
return ColorInfo.SDR_BT709_LIMITED;
}
if (inputFormat.colorInfo == null) {
Log.d(TAG, "colorInfo is null. Defaulting to SDR_BT709_LIMITED.");
return ColorInfo.SDR_BT709_LIMITED;
}
return inputFormat.colorInfo;
} }
@Nullable @Nullable
...@@ -382,12 +399,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -382,12 +399,6 @@ import org.checkerframework.dataflow.qual.Pure;
outputRotationDegrees = 90; outputRotationDegrees = 90;
} }
boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled();
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
// TODO(b/237674316): Get the color info from the decoder output media format instead.
ColorInfo outputColorInfo =
isInputToneMapped ? ColorInfo.SDR_BT709_LIMITED : inputFormat.colorInfo;
Format requestedEncoderFormat = Format requestedEncoderFormat =
new Format.Builder() new Format.Builder()
.setWidth(requestedWidth) .setWidth(requestedWidth)
...@@ -395,14 +406,14 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -395,14 +406,14 @@ import org.checkerframework.dataflow.qual.Pure;
.setRotationDegrees(0) .setRotationDegrees(0)
.setFrameRate(inputFormat.frameRate) .setFrameRate(inputFormat.frameRate)
.setSampleMimeType(requestedOutputMimeType) .setSampleMimeType(requestedOutputMimeType)
.setColorInfo(outputColorInfo) .setColorInfo(getSupportedInputColor())
.build(); .build();
encoder = encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
Format encoderSupportedFormat = encoder.getConfigurationFormat(); Format encoderSupportedFormat = encoder.getConfigurationFormat();
if (isHdrEditingEnabled()) { if (ColorInfo.isHdr(requestedEncoderFormat.colorInfo)) {
if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) { if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) {
throw createEncodingException( throw createEncodingException(
new IllegalStateException("MIME type fallback unsupported with HDR editing"), new IllegalStateException("MIME type fallback unsupported with HDR editing"),
...@@ -413,6 +424,9 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -413,6 +424,9 @@ import org.checkerframework.dataflow.qual.Pure;
encoderSupportedFormat); encoderSupportedFormat);
} }
} }
boolean isInputToneMapped =
ColorInfo.isHdr(inputFormat.colorInfo)
&& !ColorInfo.isHdr(requestedEncoderFormat.colorInfo);
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest( createFallbackTransformationRequest(
transformationRequest, transformationRequest,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment