HDR: Add effect interface support for different in/out color transfers

Modify FrameProcessor and MatrixTextureProcessor interfaces to support
different input and output color transfers. Does not implement conversion between
color ranges (ex. HDR and SDR), but should allow for conversion between color
transfers of the same color range (ex. HLG and PQ).

This supports in-app tone mapping, where we need a single FrameProcessor to
input HDR color transfers (ex. HLG/PQ) and output SDR (ex. gamma2.2). This also
supports previewing, where we need a single FrameProcessor to be able to input HLG
and output PQ.

Manually tested by confirming colors still look right on SDR and HDR videos
with a rotation and color affect applied.

PiperOrigin-RevId: 493108678

library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java

@@ -44,9 +44,11 @@ public interface FrameProcessor {
      * Creates a new {@link FrameProcessor} instance.
      *
      * @param context A {@link Context}.
-     * @param effects The {@link Effect} instances to apply to each frame.
+     * @param effects The {@link Effect} instances to apply to each frame. Applied on the {@code
+     *     outputColorInfo}'s color space.
      * @param debugViewProvider A {@link DebugViewProvider}.
-     * @param colorInfo The {@link ColorInfo} for input and output frames.
+     * @param inputColorInfo The {@link ColorInfo} for input frames.
+     * @param outputColorInfo The {@link ColorInfo} for output frames.
      * @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will render
      *     output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface}
      *     automatically as {@link FrameProcessor} is done processing them. If {@code false}, the
@@ -62,7 +64,8 @@ public interface FrameProcessor {
         Context context,
         List<Effect> effects,
         DebugViewProvider debugViewProvider,
-        ColorInfo colorInfo,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
         boolean releaseFramesAutomatically,
         Executor executor,
         Listener listener)

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java

@@ -286,7 +286,8 @@ public final class GlEffectsFrameProcessorFrameReleaseTest {
                             (context, useHdr) ->
                                 new BlankFrameProducer(inputPresentationTimesUs, useHdr)),
                     DebugViewProvider.NONE,
-                    ColorInfo.SDR_BT709_LIMITED,
+                    /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
+                    /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
                     releaseFramesAutomatically,
                     MoreExecutors.directExecutor(),
                     new FrameProcessor.Listener() {

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java

@@ -418,7 +418,8 @@ public final class GlEffectsFrameProcessorPixelTest {
                     getApplicationContext(),
                     effects,
                     DebugViewProvider.NONE,
-                    ColorInfo.SDR_BT709_LIMITED,
+                    /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
+                    /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
                     /* releaseFramesAutomatically= */ true,
                     MoreExecutors.directExecutor(),
                     new FrameProcessor.Listener() {

library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl

@@ -23,9 +23,10 @@
 // 3. Applies an EOTF based on uEotfColorTransfer, yielding optical linear
 //    BT.2020 RGB.
 // 4. Applies a 4x4 RGB color matrix to change the pixel colors.
-// 5. If uOetfColorTransfer is COLOR_TRANSFER_LINEAR, outputs linear colors as
-//    is to intermediate shaders. Otherwise, applies the HLG or PQ OETF, based
-// on uOetfColorTransfer, to provide the corresponding output electrical color.
+// 5. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for
+//    outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 /
+//    COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an
+//    encoder).
 // The output will be red if an error has occurred.
 
 #extension GL_OES_EGL_image_external : require

library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl

@@ -20,15 +20,18 @@
 // 2. Transforms the electrical colors to optical colors using the SMPTE 170M
 //    EOTF.
 // 3. Applies a 4x4 RGB color matrix to change the pixel colors.
-// 4. Transforms the optical colors back to electrical ones if uApplyOetf == 1
-//    using the SMPTE 170M OETF.
+// 4. Output as requested by uOetfColorTransfer. Use COLOR_TRANSFER_LINEAR for
+//    outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to output
+//    electrical colors via an OETF (e.g. to an encoder).
 
 #extension GL_OES_EGL_image_external : require
 precision mediump float;
 uniform samplerExternalOES uTexSampler;
 uniform mat4 uRgbMatrix;
 varying vec2 vTexSamplingCoord;
-uniform int uApplyOetf;
+// C.java#ColorTransfer value.
+// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
+uniform int uOetfColorTransfer;
 
 const float inverseGamma = 0.4500;
 const float gamma = 1.0 / inverseGamma;
@@ -61,12 +64,26 @@ float sdrOetfSingleChannel(float opticalChannel) {
 
 // Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
 vec3 sdrOetf(vec3 opticalColor) {
-  return uApplyOetf == 1
-    ? vec3(
+  return vec3(
       sdrOetfSingleChannel(opticalColor.r),
       sdrOetfSingleChannel(opticalColor.g),
-      sdrOetfSingleChannel(opticalColor.b))
-    : opticalColor;
+      sdrOetfSingleChannel(opticalColor.b));
+}
+
+// Applies the appropriate OETF to convert linear optical signals to nonlinear
+// electrical signals. Input and output are both normalized to [0, 1].
+highp vec3 applyOetf(highp vec3 linearColor) {
+  // LINT.IfChange(color_transfer_oetf)
+  const int COLOR_TRANSFER_LINEAR = 1;
+  const int COLOR_TRANSFER_SDR_VIDEO = 3;
+  if (uOetfColorTransfer == COLOR_TRANSFER_LINEAR) {
+    return linearColor;
+  } else if(uOetfColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
+    return sdrOetf(linearColor);
+  } else {
+    // Output red as an obviously visible error.
+    return vec3(1.0, 0.0, 0.0);
+  }
 }
 
 void main() {
@@ -75,5 +92,5 @@ void main() {
 
   vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
 
-  gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
+  gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
 }

library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalMatrixTextureProcessorWrapper.java

@@ -70,7 +70,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final EGLContext eglContext;
   private final DebugViewProvider debugViewProvider;
   private final boolean sampleFromExternalTexture;
-  private final ColorInfo colorInfo;
+  private final ColorInfo inputColorInfo;
+  private final ColorInfo outputColorInfo;
   private final boolean releaseFramesAutomatically;
   private final Executor frameProcessorListenerExecutor;
   private final FrameProcessor.Listener frameProcessorListener;
@@ -104,7 +105,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       ImmutableList<RgbMatrix> rgbMatrices,
       DebugViewProvider debugViewProvider,
       boolean sampleFromExternalTexture,
-      ColorInfo colorInfo,
+      ColorInfo inputColorInfo,
+      ColorInfo outputColorInfo,
       boolean releaseFramesAutomatically,
       Executor frameProcessorListenerExecutor,
       FrameProcessor.Listener frameProcessorListener) {
@@ -115,7 +117,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.eglContext = eglContext;
     this.debugViewProvider = debugViewProvider;
     this.sampleFromExternalTexture = sampleFromExternalTexture;
-    this.colorInfo = colorInfo;
+    this.inputColorInfo = inputColorInfo;
+    this.outputColorInfo = outputColorInfo;
     this.releaseFramesAutomatically = releaseFramesAutomatically;
     this.frameProcessorListenerExecutor = frameProcessorListenerExecutor;
     this.frameProcessorListener = frameProcessorListener;
@@ -336,13 +339,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
     @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
     if (outputEglSurface == null) {
-      boolean colorInfoIsHdr = ColorInfo.isTransferHdr(colorInfo);
+      boolean outputTransferIsHdr = ColorInfo.isTransferHdr(outputColorInfo);
 
       outputEglSurface =
           GlUtil.createEglSurface(
               eglDisplay,
               outputSurfaceInfo.surface,
-              colorInfoIsHdr
+              outputTransferIsHdr
                   ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
                   : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888);
 
@@ -352,7 +355,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               outputSurfaceInfo.width, outputSurfaceInfo.height);
       if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
         debugSurfaceViewWrapper =
-            new SurfaceViewWrapper(eglDisplay, eglContext, colorInfoIsHdr, debugSurfaceView);
+            new SurfaceViewWrapper(eglDisplay, eglContext, outputTransferIsHdr, debugSurfaceView);
       }
       this.debugSurfaceView = debugSurfaceView;
     }
@@ -390,12 +393,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         matrixTransformationListBuilder.build();
     if (sampleFromExternalTexture) {
       matrixTextureProcessor =
-          MatrixTextureProcessor.createWithExternalSamplerApplyingEotfThenOetf(
-              context, expandedMatrixTransformations, rgbMatrices, colorInfo);
+          MatrixTextureProcessor.createWithExternalSampler(
+              context,
+              expandedMatrixTransformations,
+              rgbMatrices,
+              /* inputColorInfo= */ inputColorInfo,
+              /* outputColorInfo= */ outputColorInfo);
     } else {
       matrixTextureProcessor =
           MatrixTextureProcessor.createApplyingOetf(
-              context, expandedMatrixTransformations, rgbMatrices, colorInfo);
+              context, expandedMatrixTransformations, rgbMatrices, outputColorInfo);
     }
 
     matrixTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);

library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java

@@ -68,12 +68,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
         Context context,
         List<Effect> effects,
         DebugViewProvider debugViewProvider,
-        ColorInfo colorInfo,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
         boolean releaseFramesAutomatically,
         Executor listenerExecutor,
         Listener listener)
         throws FrameProcessingException {
       // TODO(b/261188041) Add tests to verify the Listener is invoked on the given Executor.
+
+      // TODO(b/239735341): Reduce the scope of these checks by implementing GL tone-mapping.
+      checkArgument(
+          inputColorInfo.colorSpace == outputColorInfo.colorSpace,
+          "Conversion between HDR and SDR color spaces is not yet supported.");
+      checkArgument(
+          ColorInfo.isTransferHdr(inputColorInfo) == ColorInfo.isTransferHdr(outputColorInfo),
+          "Conversion between HDR and SDR color transfers is not yet supported.");
+      checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
+      checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
+
       ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
 
       Future<GlEffectsFrameProcessor> glFrameProcessorFuture =
@@ -83,7 +95,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
                       context,
                       effects,
                       debugViewProvider,
-                      colorInfo,
+                      inputColorInfo,
+                      outputColorInfo,
                       releaseFramesAutomatically,
                       singleThreadExecutorService,
                       listenerExecutor,
@@ -115,7 +128,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       Context context,
       List<Effect> effects,
       DebugViewProvider debugViewProvider,
-      ColorInfo colorInfo,
+      ColorInfo inputColorInfo,
+      ColorInfo outputColorInfo,
       boolean releaseFramesAutomatically,
       ExecutorService singleThreadExecutorService,
       Executor executor,
@@ -125,10 +139,11 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
 
     // TODO(b/237674316): Delay initialization of things requiring the colorInfo, to
     //  configure based on the color info from the decoder output media format instead.
-    boolean useHdr = ColorInfo.isTransferHdr(colorInfo);
     EGLDisplay eglDisplay = GlUtil.createEglDisplay();
     int[] configAttributes =
-        useHdr ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
+        ColorInfo.isTransferHdr(outputColorInfo)
+            ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
+            : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
     EGLContext eglContext = GlUtil.createEglContext(eglDisplay, configAttributes);
     GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes);
 
@@ -139,7 +154,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
             eglDisplay,
             eglContext,
             debugViewProvider,
-            colorInfo,
+            inputColorInfo,
+            outputColorInfo,
             releaseFramesAutomatically,
             executor,
             listener);
@@ -173,7 +189,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       EGLDisplay eglDisplay,
       EGLContext eglContext,
       DebugViewProvider debugViewProvider,
-      ColorInfo colorInfo,
+      ColorInfo inputColorInfo,
+      ColorInfo outputColorInfo,
       boolean releaseFramesAutomatically,
       Executor executor,
       Listener listener)
@@ -184,6 +201,9 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
         new ImmutableList.Builder<>();
     ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
     boolean sampleFromExternalTexture = true;
+    ColorInfo linearColorInfo =
+        new ColorInfo(
+            inputColorInfo.colorSpace, inputColorInfo.colorRange, C.COLOR_TRANSFER_LINEAR, null);
     for (int i = 0; i < effects.size(); i++) {
       Effect effect = effects.get(i);
       checkArgument(effect instanceof GlEffect, "GlEffectsFrameProcessor only supports GlEffects");
@@ -203,24 +223,28 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
       ImmutableList<GlMatrixTransformation> matrixTransformations =
           matrixTransformationListBuilder.build();
       ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
+      boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
       if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) {
         MatrixTextureProcessor matrixTextureProcessor;
         if (sampleFromExternalTexture) {
           matrixTextureProcessor =
-              MatrixTextureProcessor.createWithExternalSamplerApplyingEotf(
-                  context, matrixTransformations, rgbMatrices, colorInfo);
+              MatrixTextureProcessor.createWithExternalSampler(
+                  context,
+                  matrixTransformations,
+                  rgbMatrices,
+                  /* inputColorInfo= */ inputColorInfo,
+                  /* outputColorInfo= */ linearColorInfo);
         } else {
           matrixTextureProcessor =
               MatrixTextureProcessor.create(
-                  context, matrixTransformations, rgbMatrices, ColorInfo.isTransferHdr(colorInfo));
+                  context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
         }
         textureProcessorListBuilder.add(matrixTextureProcessor);
         matrixTransformationListBuilder = new ImmutableList.Builder<>();
         rgbMatrixListBuilder = new ImmutableList.Builder<>();
         sampleFromExternalTexture = false;
       }
-      textureProcessorListBuilder.add(
-          glEffect.toGlTextureProcessor(context, ColorInfo.isTransferHdr(colorInfo)));
+      textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, isOutputTransferHdr));
     }
 
     textureProcessorListBuilder.add(
@@ -232,7 +256,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
             rgbMatrixListBuilder.build(),
             debugViewProvider,
             sampleFromExternalTexture,
-            colorInfo,
+            /* inputColorInfo= */ sampleFromExternalTexture ? inputColorInfo : linearColorInfo,
+            outputColorInfo,
             releaseFramesAutomatically,
             executor,
             listener));

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -155,17 +155,19 @@ import org.checkerframework.dataflow.qual.Pure;
             transformationRequest,
             fallbackListener);
 
+    // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
+    // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
+    // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
+    // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
+    ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor();
     try {
       frameProcessor =
           frameProcessorFactory.create(
               context,
               effectsListBuilder.build(),
               debugViewProvider,
-              // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
-              // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
-              // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
-              // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
-              encoderWrapper.getSupportedInputColor(),
+              /* inputColorInfo= */ encoderSupportedInputColor,
+              /* outputColorInfo= */ encoderSupportedInputColor,
               /* releaseFramesAutomatically= */ true,
               MoreExecutors.directExecutor(),
               new FrameProcessor.Listener() {