HDR: Implement PQ to SDR tone-mapping.

Tested manually on the Pixel 7 and Samsung S10.

PiperOrigin-RevId: 501626354

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java

@@ -88,13 +88,20 @@ public final class GlEffectsFrameProcessorPixelTest {
       "media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png";
       "media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png";
   // This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate
   // This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate
   // this file.
   // this file.
-  public static final String TONE_MAP_HDR_TO_SDR_PNG_ASSET_PATH =
-      "media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_hdr_to_sdr.png";
+  public static final String TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_hlg_to_sdr.png";
+  // This file is generated on a Pixel 7, because the emulator isn't able to decode PQ to generate
+  // this file.
+  public static final String TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_pq_to_sdr.png";
 
 
   /** Input video of which we only use the first frame. */
   /** Input video of which we only use the first frame. */
   private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
   private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
   /** Input HLG video of which we only use the first frame. */
   /** Input HLG video of which we only use the first frame. */
   private static final String INPUT_HLG_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
   private static final String INPUT_HLG_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
+  /** Input PQ video of which we only use the first frame. */
+  private static final String INPUT_PQ_MP4_ASSET_STRING = "media/mp4/hdr10-1080p.mp4";
+
   /**
   /**
    * Time to wait for the decoded frame to populate the {@link GlEffectsFrameProcessor} instance's
    * Time to wait for the decoded frame to populate the {@link GlEffectsFrameProcessor} instance's
    * input surface and the {@link GlEffectsFrameProcessor} to finish processing the frame, in
    * input surface and the {@link GlEffectsFrameProcessor} to finish processing the frame, in
@@ -399,24 +406,58 @@ public final class GlEffectsFrameProcessorPixelTest {
     // TODO(b/239735341): Move this test to mobileharness testing.
     // TODO(b/239735341): Move this test to mobileharness testing.
     String testId = "drawHlgFrame_toneMap";
     String testId = "drawHlgFrame_toneMap";
     ColorInfo hlgColor =
     ColorInfo hlgColor =
-        new ColorInfo(
-            C.COLOR_SPACE_BT2020,
-            C.COLOR_RANGE_LIMITED,
-            C.COLOR_TRANSFER_HLG,
-            /* hdrStaticInfo= */ null);
+        new ColorInfo.Builder()
+            .setColorSpace(C.COLOR_SPACE_BT2020)
+            .setColorRange(C.COLOR_RANGE_LIMITED)
+            .setColorTransfer(C.COLOR_TRANSFER_HLG)
+            .build();
     ColorInfo toneMapSdrColor =
     ColorInfo toneMapSdrColor =
-        new ColorInfo(
-            C.COLOR_SPACE_BT709,
-            C.COLOR_RANGE_LIMITED,
-            C.COLOR_TRANSFER_GAMMA_2_2,
-            /* hdrStaticInfo= */ null);
+        new ColorInfo.Builder()
+            .setColorSpace(C.COLOR_SPACE_BT709)
+            .setColorRange(C.COLOR_RANGE_LIMITED)
+            .setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
+            .build();
     setUpAndPrepareFirstFrame(
     setUpAndPrepareFirstFrame(
         INPUT_HLG_MP4_ASSET_STRING,
         INPUT_HLG_MP4_ASSET_STRING,
         DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
         DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
         /* inputColorInfo= */ hlgColor,
         /* inputColorInfo= */ hlgColor,
         /* outputColorInfo= */ toneMapSdrColor,
         /* outputColorInfo= */ toneMapSdrColor,
         /* effects= */ ImmutableList.of());
         /* effects= */ ImmutableList.of());
-    Bitmap expectedBitmap = readBitmap(TONE_MAP_HDR_TO_SDR_PNG_ASSET_PATH);
+    Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH);
+
+    Bitmap actualBitmap = processFirstFrameAndEnd();
+
+    maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
+    // TODO(b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  @Ignore("b/261877288 Test can only run on physical devices because decoder can't decode PQ.")
+  public void drawPqFrame_toneMap_producesExpectedOutput() throws Exception {
+    // TODO(b/239735341): Move this test to mobileharness testing.
+    String testId = "drawPqFrame_toneMap";
+    ColorInfo pqColor =
+        new ColorInfo.Builder()
+            .setColorSpace(C.COLOR_SPACE_BT2020)
+            .setColorRange(C.COLOR_RANGE_LIMITED)
+            .setColorTransfer(C.COLOR_TRANSFER_ST2084)
+            .build();
+    ColorInfo toneMapSdrColor =
+        new ColorInfo.Builder()
+            .setColorSpace(C.COLOR_SPACE_BT709)
+            .setColorRange(C.COLOR_RANGE_LIMITED)
+            .setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
+            .build();
+    setUpAndPrepareFirstFrame(
+        INPUT_PQ_MP4_ASSET_STRING,
+        DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
+        /* inputColorInfo= */ pqColor,
+        /* outputColorInfo= */ toneMapSdrColor,
+        /* effects= */ ImmutableList.of());
+    Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH);
 
 
     Bitmap actualBitmap = processFirstFrameAndEnd();
     Bitmap actualBitmap = processFirstFrameAndEnd();
 
 

library/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl

@@ -48,6 +48,12 @@ uniform int uOutputColorTransfer;
 in vec2 vTexSamplingCoord;
 in vec2 vTexSamplingCoord;
 out vec4 outColor;
 out vec4 outColor;
 
 
+// LINT.IfChange(color_transfer)
+const int COLOR_TRANSFER_LINEAR = 1;
+const int COLOR_TRANSFER_GAMMA_2_2 = 10;
+const int COLOR_TRANSFER_ST2084 = 6;
+const int COLOR_TRANSFER_HLG = 7;
+
 // TODO(b/227624622): Consider using mediump to save precision, if it won't lead
 // TODO(b/227624622): Consider using mediump to save precision, if it won't lead
 //  to noticeable quantization errors.
 //  to noticeable quantization errors.
 
 
@@ -93,10 +99,6 @@ highp vec3 pqEotf(highp vec3 pqColor) {
 // Applies the appropriate EOTF to convert nonlinear electrical values to linear
 // Applies the appropriate EOTF to convert nonlinear electrical values to linear
 // optical values. Input and output are both normalized to [0, 1].
 // optical values. Input and output are both normalized to [0, 1].
 highp vec3 applyEotf(highp vec3 electricalColor) {
 highp vec3 applyEotf(highp vec3 electricalColor) {
-  // LINT.IfChange(color_transfer)
-  const int COLOR_TRANSFER_ST2084 = 6;
-  const int COLOR_TRANSFER_HLG = 7;
-
   if (uInputColorTransfer == COLOR_TRANSFER_ST2084) {
   if (uInputColorTransfer == COLOR_TRANSFER_ST2084) {
     return pqEotf(electricalColor);
     return pqEotf(electricalColor);
   } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) {
   } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) {
@@ -136,6 +138,25 @@ highp vec3 applyHlgBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
   return linearRgbBt709;
   return linearRgbBt709;
 }
 }
 
 
+// Apply the PQ BT2020 to BT709 OOTF.
+highp vec3 applyPqBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
+  float pqPeakLuminance = 10000.0;
+  float sdrPeakLuminance = 500.0;
+
+  return linearRgbBt2020 * pqPeakLuminance / sdrPeakLuminance;
+}
+
+highp vec3 applyBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
+  if (uInputColorTransfer == COLOR_TRANSFER_ST2084) {
+    return applyPqBt2020ToBt709Ootf(linearRgbBt2020);
+  } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) {
+    return applyHlgBt2020ToBt709Ootf(linearRgbBt2020);
+  } else {
+    // Output red as an obviously visible error.
+    return vec3(1.0, 0.0, 0.0);
+  }
+}
+
 // BT.2100 / BT.2020 HLG OETF for one channel.
 // BT.2100 / BT.2020 HLG OETF for one channel.
 highp float hlgOetfSingleChannel(highp float linearChannel) {
 highp float hlgOetfSingleChannel(highp float linearChannel) {
   // Specification:
   // Specification:
@@ -194,11 +215,6 @@ vec3 gamma22Oetf(highp vec3 linearColor) {
 // Applies the appropriate OETF to convert linear optical signals to nonlinear
 // Applies the appropriate OETF to convert linear optical signals to nonlinear
 // electrical signals. Input and output are both normalized to [0, 1].
 // electrical signals. Input and output are both normalized to [0, 1].
 highp vec3 applyOetf(highp vec3 linearColor) {
 highp vec3 applyOetf(highp vec3 linearColor) {
-  // LINT.IfChange(color_transfer_oetf)
-  const int COLOR_TRANSFER_LINEAR = 1;
-  const int COLOR_TRANSFER_GAMMA_2_2 = 10;
-  const int COLOR_TRANSFER_ST2084 = 6;
-  const int COLOR_TRANSFER_HLG = 7;
   if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) {
   if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) {
     return pqOetf(linearColor);
     return pqOetf(linearColor);
   } else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) {
   } else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) {
@@ -221,9 +237,8 @@ vec3 yuvToRgb(vec3 yuv) {
 void main() {
 void main() {
   vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
   vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
   vec3 opticalColorBt2020 = applyEotf(yuvToRgb(srcYuv));
   vec3 opticalColorBt2020 = applyEotf(yuvToRgb(srcYuv));
-  // TODO(b/239735341): Add support for PQ tone-mapping.
   vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1)
   vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1)
-      ? vec4(applyHlgBt2020ToBt709Ootf(opticalColorBt2020), 1.0)
+      ? vec4(applyBt2020ToBt709Ootf(opticalColorBt2020), 1.0)
       : vec4(opticalColorBt2020, 1.0);
       : vec4(opticalColorBt2020, 1.0);
   vec4 transformedColors = uRgbMatrix * opticalColor;
   vec4 transformedColors = uRgbMatrix * opticalColor;
   outColor = vec4(applyOetf(transformedColors.rgb), 1.0);
   outColor = vec4(applyOetf(transformedColors.rgb), 1.0);

library/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl

@@ -74,7 +74,7 @@ vec3 smpte170mOetf(vec3 opticalColor) {
 // Applies the appropriate OETF to convert linear optical signals to nonlinear
 // Applies the appropriate OETF to convert linear optical signals to nonlinear
 // electrical signals. Input and output are both normalized to [0, 1].
 // electrical signals. Input and output are both normalized to [0, 1].
 highp vec3 applyOetf(highp vec3 linearColor) {
 highp vec3 applyOetf(highp vec3 linearColor) {
-  // LINT.IfChange(color_transfer_oetf)
+  // LINT.IfChange(color_transfer)
   const int COLOR_TRANSFER_LINEAR = 1;
   const int COLOR_TRANSFER_LINEAR = 1;
   const int COLOR_TRANSFER_SDR_VIDEO = 3;
   const int COLOR_TRANSFER_SDR_VIDEO = 3;
   if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
   if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {

library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java

@@ -90,16 +90,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
 
 
       if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
       if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
           || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
           || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
-        // GL Tone mapping is only implemented for BT2020 to BT709 and HLG to SDR (Gamma 2.2).
+        // GL Tone mapping is only implemented for BT2020 to BT709 and HDR to SDR (Gamma 2.2).
         // Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's
         // Gamma 2.2 is used instead of SMPTE 170M for SDR, despite MediaFormat's
         // COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match
         // COLOR_TRANSFER_SDR_VIDEO being defined as SMPTE 170M. This is to match
         // other known tone-mapping behavior within the Android ecosystem.
         // other known tone-mapping behavior within the Android ecosystem.
         // TODO(b/239735341): Consider migrating SDR outside tone-mapping from SMPTE
         // TODO(b/239735341): Consider migrating SDR outside tone-mapping from SMPTE
         //  170M to gamma 2.2.
         //  170M to gamma 2.2.
-        // TODO(b/239735341): Implement PQ tone-mapping to reduce the scope of these checks.
         checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
         checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
         checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020);
         checkArgument(outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020);
-        checkArgument(inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG);
+        checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
         checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
         checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
       }
       }
 
 

library/effect/src/main/java/com/google/android/exoplayer2/effect/MatrixTextureProcessor.java

@@ -217,11 +217,8 @@ import java.util.List;
               ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
               ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
               : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
               : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
 
 
-      @C.ColorTransfer int inputColorTransfer = inputColorInfo.colorTransfer;
-      checkArgument(
-          inputColorTransfer == C.COLOR_TRANSFER_HLG
-              || inputColorTransfer == C.COLOR_TRANSFER_ST2084);
-      glProgram.setIntUniform("uInputColorTransfer", inputColorTransfer);
+      checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
+      glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
       // TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
       // TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
       glProgram.setIntUniform(
       glProgram.setIntUniform(
           "uApplyHdrToSdrToneMapping",
           "uApplyHdrToSdrToneMapping",

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java

@@ -16,6 +16,8 @@
 package com.google.android.exoplayer2.transformer.mh;
 package com.google.android.exoplayer2.transformer.mh;
 
 
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_4_SECOND_HDR10;
+import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_4_SECOND_HDR10_FORMAT;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped;
 import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped;
@@ -97,4 +99,54 @@ public class ToneMapHdrToSdrUsingOpenGlTest {
       }
       }
     }
     }
   }
   }
+
+  @Test
+  public void transform_toneMap_hdr10File_toneMapsOrThrows() throws Exception {
+    String testId = "transform_glToneMap_hdr10File_toneMapsOrThrows";
+
+    if (Util.SDK_INT < 29) {
+      recordTestSkipped(
+          ApplicationProvider.getApplicationContext(),
+          testId,
+          /* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
+      return;
+    }
+
+    if (!GlUtil.isYuvTargetExtensionSupported()) {
+      recordTestSkipped(
+          getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
+      return;
+    }
+
+    if (AndroidTestUtil.skipAndLogIfInsufficientCodecSupport(
+        getApplicationContext(),
+        testId,
+        /* decodingFormat= */ MP4_ASSET_1080P_4_SECOND_HDR10_FORMAT,
+        /* encodingFormat= */ null)) {
+      return;
+    }
+
+    Context context = ApplicationProvider.getApplicationContext();
+
+    Transformer transformer =
+        new Transformer.Builder(context)
+            .setTransformationRequest(
+                new TransformationRequest.Builder()
+                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
+                    .build())
+            .build();
+    try {
+      TransformationTestResult transformationTestResult =
+          new TransformerAndroidTestRunner.Builder(context, transformer)
+              .build()
+              .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_1080P_4_SECOND_HDR10)));
+      Log.i(TAG, "Tone mapped.");
+      assertFileHasColorTransfer(transformationTestResult.filePath, C.COLOR_TRANSFER_SDR);
+    } catch (TransformationException exception) {
+      Log.i(TAG, checkNotNull(exception.getCause()).toString());
+      if (exception.errorCode != TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED) {
+        throw exception;
+      }
+    }
+  }
 }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java

@@ -77,7 +77,7 @@ public final class TransformationRequest {
    * Tone map HDR input to SDR before processing, to generate SDR output, using an OpenGL
    * Tone map HDR input to SDR before processing, to generate SDR output, using an OpenGL
    * tone-mapper.
    * tone-mapper.
    *
    *
-   * <p>Supported on API 29+, for HLG input.
+   * <p>Supported on API 29+.
    *
    *
    * <p>This may exhibit mild differences from {@link
    * <p>This may exhibit mild differences from {@link
    * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, depending on the device's tone-mapping
    * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, depending on the device's tone-mapping
@@ -86,7 +86,6 @@ public final class TransformationRequest {
    *
    *
    * <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
    * <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
    */
    */
-  // TODO(b/239735341): Implement PQ tone-mapping to remove the HLG reference.
   public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL = 2;
   public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL = 2;
   /**
   /**
    * Interpret HDR input as SDR, likely with a washed out look.
    * Interpret HDR input as SDR, likely with a washed out look.