Effect: Add support for disabling color transfers for SDR

PiperOrigin-RevId: 521805477

Effect: Add support for disabling color transfers for SDR
PiperOrigin-RevId: 521805477
2fa2c341 · tofunmi · Marc Baechinger · dfc69dee · 2fa2c341 · 2fa2c341
Commit 2fa2c341 authored Apr 04, 2023 by tofunmi Committed by Marc Baechinger Apr 05, 2023
Showing with 145 additions and 20 deletions
library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultShaderProgram.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java
--- a/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
+++ b/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
@@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
 // C.java#ColorTransfer value.
 // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
 uniform int uOutputColorTransfer;
+uniform int uEnableColorTransfer;
 const float inverseGamma = 0.4500;
 const float gamma = 1.0 / inverseGamma;
+const int GL_FALSE = 0;
+const int GL_TRUE = 1;
 // Transforms a single channel from electrical to optical SDR using the SMPTE 
 // 170M OETF.
@@ -77,7 +80,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
  // LINT.IfChange(color_transfer)
  const int COLOR_TRANSFER_LINEAR = 1;
  const int COLOR_TRANSFER_SDR_VIDEO = 3;
-  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
+  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
+    || uEnableColorTransfer == GL_FALSE) {
    return linearColor;
  } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
    return smpte170mOetf(linearColor);
@@ -87,9 +91,20 @@ highp vec3 applyOetf(highp vec3 linearColor) {
  }
 }
+vec3 applyEotf(vec3 electricalColor){
+  if (uEnableColorTransfer == GL_TRUE){
+    return smpte170mEotf(electricalColor);
+  } else if (uEnableColorTransfer == GL_FALSE) {
+    return electricalColor;
+  } else {
+    // Output blue as an obviously visible error.
+    return vec3(0.0, 0.0, 1.0);
+  }
+}
 void main() {
  vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
-  vec3 linearInputColor = smpte170mEotf(inputColor.rgb);
+  vec3 linearInputColor = applyEotf(inputColor.rgb);
  vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);

--- a/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl
+++ b/library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl
@@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
 // C.java#ColorTransfer value.
 // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
 uniform int uOutputColorTransfer;
+uniform int uEnableColorTransfer;
 const float inverseGamma = 0.4500;
 const float gamma = 1.0 / inverseGamma;
+const int GL_FALSE = 0;
+const int GL_TRUE = 1;
 // Transforms a single channel from electrical to optical SDR using the sRGB
 // EOTF.
@@ -78,7 +81,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
  // LINT.IfChange(color_transfer)
  const int COLOR_TRANSFER_LINEAR = 1;
  const int COLOR_TRANSFER_SDR_VIDEO = 3;
-  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
+  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
+    || uEnableColorTransfer == GL_FALSE) {
    return linearColor;
  } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
    return smpte170mOetf(linearColor);
@@ -88,6 +92,17 @@ highp vec3 applyOetf(highp vec3 linearColor) {
  }
 }
+vec3 applyEotf(vec3 electricalColor){
+  if (uEnableColorTransfer == GL_TRUE){
+    return srgbEotf(electricalColor) ;
+  } else if (uEnableColorTransfer == GL_FALSE) {
+    return electricalColor;
+  } else {
+    // Output blue as an obviously visible error.
+    return vec3(0.0, 0.0, 1.0);
+  }
+}
 void main() {
  vec2 vTexSamplingCoordFlipped =
      vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y);
@@ -96,7 +111,7 @@ void main() {
  // texture gets flipped. We flip the texture vertically to ensure the
  // orientation of the output is correct.
  vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoordFlipped);
-  vec3 linearInputColor = srgbEotf(inputColor.rgb);
+  vec3 linearInputColor = applyEotf(inputColor.rgb);
  vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultShaderProgram.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultShaderProgram.java
@@ -90,6 +90,9 @@ import java.util.List;
    1.6853f, -0.6530f, 0.0000f,
  };
+  private static final int GL_FALSE = 0;
+  private static final int GL_TRUE = 1;
  /** The {@link MatrixTransformation MatrixTransformations} to apply. */
  private final ImmutableList<GlMatrixTransformation> matrixTransformations;
  /** The {@link RgbMatrix RgbMatrices} to apply. */
@@ -183,6 +186,8 @@ import java.util.List;
   * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
   *     If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
   *     ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
+   * @param enableColorTransfers Whether to transfer colors to an intermediate color space when
+   *     applying effects. If the input or output is HDR, this must be {@code true}.
   * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
   *     OpenGL operation fails or is unsupported.
   */
@@ -191,7 +196,8 @@ import java.util.List;
      List<GlMatrixTransformation> matrixTransformations,
      List<RgbMatrix> rgbMatrices,
      ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
      throws VideoFrameProcessingException {
    checkState(
        !ColorInfo.isTransferHdr(inputColorInfo),
@@ -202,7 +208,12 @@ import java.util.List;
            VERTEX_SHADER_TRANSFORMATION_PATH,
            FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH);
    return createWithSampler(
-        glProgram, matrixTransformations, rgbMatrices, inputColorInfo, outputColorInfo);
+        glProgram,
+        matrixTransformations,
+        rgbMatrices,
+        inputColorInfo,
+        outputColorInfo,
+        enableColorTransfers);
  }
  /**
@@ -227,6 +238,8 @@ import java.util.List;
   * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
   *     If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
   *     ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
+   * @param enableColorTransfers Whether to transfer colors to an intermediate color space when
+   *     applying effects. If the input or output is HDR, this must be {@code true}.
   * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
   *     OpenGL operation fails or is unsupported.
   */
@@ -235,7 +248,8 @@ import java.util.List;
      List<GlMatrixTransformation> matrixTransformations,
      List<RgbMatrix> rgbMatrices,
      ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
      throws VideoFrameProcessingException {
    boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
    String vertexShaderFilePath =
@@ -251,7 +265,8 @@ import java.util.List;
        matrixTransformations,
        rgbMatrices,
        inputColorInfo,
-        outputColorInfo);
+        outputColorInfo,
+        enableColorTransfers);
  }
  /**
@@ -314,12 +329,15 @@ import java.util.List;
      List<GlMatrixTransformation> matrixTransformations,
      List<RgbMatrix> rgbMatrices,
      ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
      throws VideoFrameProcessingException {
+    glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE);
    boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
    @C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
    if (isInputTransferHdr) {
      checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
+      checkArgument(enableColorTransfers);
      // In HDR editing mode the decoder output is sampled in YUV.
      if (!GlUtil.isYuvTargetExtensionSupported()) {

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
@@ -46,6 +46,7 @@ import com.google.android.exoplayer2.util.VideoFrameProcessor;
 import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -62,9 +63,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
  /** A factory for {@link DefaultVideoFrameProcessor} instances. */
  public static final class Factory implements VideoFrameProcessor.Factory {
+    /** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */
+    public static final class Builder {
+      private boolean enableColorTransfers;
+      /** Creates an instance. */
+      public Builder() {
+        enableColorTransfers = true;
+      }
+      /**
+       * Sets whether to transfer colors to an intermediate color space when applying effects.
+       *
+       * <p>If the input or output is HDR, this must be {@code true}.
+       */
+      @CanIgnoreReturnValue
+      public DefaultVideoFrameProcessor.Factory.Builder setEnableColorTransfers(
+          boolean enableColorTransfers) {
+        this.enableColorTransfers = enableColorTransfers;
+        return this;
+      }
+      /** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
+      public DefaultVideoFrameProcessor.Factory build() {
+        return new DefaultVideoFrameProcessor.Factory(enableColorTransfers);
+      }
+    }
+    /** Whether to transfer colors to an intermediate color space when applying effects. */
+    public final boolean enableColorTransfers;
    private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
    private boolean outputToTexture;
+    private Factory(boolean enableColorTransfers) {
+      this.enableColorTransfers = enableColorTransfers;
+    }
+    // TODO(276913828): Remove and change all calls to a builder.
+    /**
+     * @deprecated Use {@link DefaultVideoFrameProcessor.Factory.Builder} instead.
+     */
+    @Deprecated
+    public Factory() {
+      this(/* enableColorTransfers= */ true);
+    }
+    // TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
    /**
     * {@inheritDoc}
     *
@@ -77,10 +123,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      return this;
    }
+    // TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
    /**
     * Sets whether to output to a texture for testing.
     *
-     * <p>Must be called before {@link #create}.
+     * <p>Must be called before {@link VideoFrameProcessor.Factory#create}.
     *
     * <p>The default value is {@code false}.
     */
@@ -112,6 +159,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
     * are HDR}, textures will use {@link GLES30#GL_RGBA16F} and {@link GLES30#GL_HALF_FLOAT}.
     * Otherwise, textures will use {@link GLES20#GL_RGBA} and {@link GLES20#GL_UNSIGNED_BYTE}.
     *
+     * <p>If {@code inputColorInfo} or {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr}
+     * are HDR}, color transfers must be enabled.
+     *
     * <p>If {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr is HDR}, the context will
     * be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_1010102}. Otherwise, the context
     * will be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_8888}.
@@ -137,6 +187,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
      checkArgument(outputColorInfo.isValid());
      checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
+      if (ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) {
+        checkArgument(enableColorTransfers);
+      }
      if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
          || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
@@ -161,6 +214,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
                      debugViewProvider,
                      inputColorInfo,
                      outputColorInfo,
+                      enableColorTransfers,
                      isInputTextureExternal,
                      releaseFramesAutomatically,
                      singleThreadExecutorService,
@@ -405,6 +459,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      DebugViewProvider debugViewProvider,
      ColorInfo inputColorInfo,
      ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
      boolean isInputTextureExternal,
      boolean releaseFramesAutomatically,
      ExecutorService singleThreadExecutorService,
@@ -450,6 +505,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
            debugViewProvider,
            inputColorInfo,
            outputColorInfo,
+            enableColorTransfers,
            isInputTextureExternal,
            releaseFramesAutomatically,
            executor,
@@ -490,6 +546,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      DebugViewProvider debugViewProvider,
      ColorInfo inputColorInfo,
      ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
      boolean isInputTextureExternal,
      boolean releaseFramesAutomatically,
      Executor executor,
@@ -535,11 +592,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
          if (isInputTextureExternal) {
            defaultShaderProgram =
                DefaultShaderProgram.createWithExternalSampler(
-                    context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
+                    context,
+                    matrixTransformations,
+                    rgbMatrices,
+                    inputColorInfo,
+                    linearColorInfo,
+                    enableColorTransfers);
          } else {
            defaultShaderProgram =
                DefaultShaderProgram.createWithInternalSampler(
-                    context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
+                    context,
+                    matrixTransformations,
+                    rgbMatrices,
+                    inputColorInfo,
+                    linearColorInfo,
+                    enableColorTransfers);
          }
        } else {
          defaultShaderProgram =
@@ -564,6 +631,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
            debugViewProvider,
            /* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
            outputColorInfo,
+            enableColorTransfers,
            sampleFromInputTexture,
            isInputTextureExternal,
            releaseFramesAutomatically,

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java
@@ -77,6 +77,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  private final boolean isInputTextureExternal;
  private final ColorInfo inputColorInfo;
  private final ColorInfo outputColorInfo;
+  private final boolean enableColorTransfers;
  private final boolean releaseFramesAutomatically;
  private final Executor videoFrameProcessorListenerExecutor;
  private final VideoFrameProcessor.Listener videoFrameProcessorListener;
@@ -115,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
      DebugViewProvider debugViewProvider,
      ColorInfo inputColorInfo,
      ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
      boolean sampleFromInputTexture,
      boolean isInputTextureExternal,
      boolean releaseFramesAutomatically,
@@ -132,6 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    this.isInputTextureExternal = isInputTextureExternal;
    this.inputColorInfo = inputColorInfo;
    this.outputColorInfo = outputColorInfo;
+    this.enableColorTransfers = enableColorTransfers;
    this.releaseFramesAutomatically = releaseFramesAutomatically;
    this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
    this.videoFrameProcessorListener = videoFrameProcessorListener;
@@ -491,7 +494,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                expandedMatrixTransformations,
                rgbMatrices,
                inputColorInfo,
-                outputColorInfo);
+                outputColorInfo,
+                enableColorTransfers);
      } else {
        defaultShaderProgram =
            DefaultShaderProgram.createWithInternalSampler(
@@ -499,7 +503,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                expandedMatrixTransformations,
                rgbMatrices,
                inputColorInfo,
-                outputColorInfo);
+                outputColorInfo,
+                enableColorTransfers);
      }
    } else {
      defaultShaderProgram =
@@ -522,12 +527,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
          .maybeRenderToSurfaceView(
              () -> {
                GlUtil.clearOutputFrame();
-                @C.ColorTransfer
+                if (enableColorTransfers) {
-                int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
+                  defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
-                defaultShaderProgram.setOutputColorTransfer(
+                } else {
-                    debugSurfaceViewWrapper.outputColorTransfer);
+                  @C.ColorTransfer
-                defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
+                  int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
-                defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
+                  defaultShaderProgram.setOutputColorTransfer(
+                      debugSurfaceViewWrapper.outputColorTransfer);
+                  defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
+                  defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
+                }
              },
              glObjectsProvider);
    } catch (VideoFrameProcessingException | GlUtil.GlException e) {