HDR: Implement Transformer HDR to SDR GL tone-mapping API

Note that we simply use GlEffectsFrameProcessor in-app / GL tone-mapping, so PQ->SDR tone-mapping isn't yet implemented.

Tested manually using the demo on Pixel 7, to confirm that device and in-app tone
mapping behave similarly.

PiperOrigin-RevId: 496700231

demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java

@@ -176,7 +176,12 @@ public final class ConfigurationActivity extends AppCompatActivity {
       HDR_MODE_DESCRIPTIONS =
           new ImmutableMap.Builder<String, @TransformationRequest.HdrMode Integer>()
               .put("Keep HDR", TransformationRequest.HDR_MODE_KEEP_HDR)
-              .put("Tone-map HDR to SDR", TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
+              .put(
+                  "MediaCodec tone-map HDR to SDR",
+                  TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
+              .put(
+                  "OpenGL tone-map HDR to SDR",
+                  TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
               .put(
                   "Force Interpret HDR as SDR",
                   TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR)

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/HdrEditingTest.java

@@ -183,7 +183,7 @@ public class HdrEditingTest {
                         .isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR);
                     isToneMappingFallbackApplied.set(
                         fallbackTransformationRequest.hdrMode
-                            == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR);
+                            == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
                   }
                 })
             .build();
@@ -236,7 +236,7 @@ public class HdrEditingTest {
                         .isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR);
                     isToneMappingFallbackApplied.set(
                         fallbackTransformationRequest.hdrMode
-                            == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR);
+                            == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
                   }
                 })
             .build();

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/HdrToSdrToneMapTest.java → library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/ToneMapHdrToSdrUsingMediaCodecTest.java

@@ -38,11 +38,12 @@ import org.junit.runner.RunWith;
 
 /**
  * {@link Transformer} instrumentation test for applying an {@linkplain
- * TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR HDR to SDR tone mapping edit}.
+ * TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC HDR to SDR tone mapping
+ * edit}.
  */
 @RunWith(AndroidJUnit4.class)
-public class HdrToSdrToneMapTest {
-  public static final String TAG = "HdrToSdrToneMapTest";
+public class ToneMapHdrToSdrUsingMediaCodecTest {
+  public static final String TAG = "ToneMapHdrToSdrUsingMediaCodecTest";
 
   @Test
   public void transform_toneMapNoRequestedTranscode_hdr10File_toneMapsOrThrows() throws Exception {
@@ -53,7 +54,7 @@ public class HdrToSdrToneMapTest {
         new Transformer.Builder(context)
             .setTransformationRequest(
                 new TransformationRequest.Builder()
-                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
+                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
                     .build())
             .addListener(
                 new Transformer.Listener() {
@@ -98,7 +99,7 @@ public class HdrToSdrToneMapTest {
         new Transformer.Builder(context)
             .setTransformationRequest(
                 new TransformationRequest.Builder()
-                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
+                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
                     .build())
             .addListener(
                 new Transformer.Listener() {
@@ -143,7 +144,7 @@ public class HdrToSdrToneMapTest {
         new Transformer.Builder(context)
             .setTransformationRequest(
                 new TransformationRequest.Builder()
-                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
+                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
                     .setRotationDegrees(180)
                     .build())
             .addListener(
@@ -189,7 +190,7 @@ public class HdrToSdrToneMapTest {
         new Transformer.Builder(context)
             .setTransformationRequest(
                 new TransformationRequest.Builder()
-                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
+                    .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
                     .setRotationDegrees(180)
                     .build())
             .addListener(

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java

@@ -38,7 +38,8 @@ public final class TransformationRequest {
   /**
    * The strategy to use to transcode or edit High Dynamic Range (HDR) input video.
    *
-   * <p>One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR}, or {@link
+   * <p>One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC},
+   * {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL}, or {@link
    * #HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}.
    *
    * <p>Standard Dynamic Range (SDR) input video is unaffected by these settings.
@@ -48,8 +49,9 @@ public final class TransformationRequest {
   @Target(TYPE_USE)
   @IntDef({
     HDR_MODE_KEEP_HDR,
-    HDR_MODE_TONE_MAP_HDR_TO_SDR,
-    HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR
+    HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC,
+    HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL,
+    HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR,
   })
   public @interface HdrMode {}
   /**
@@ -58,26 +60,43 @@ public final class TransformationRequest {
    * <p>Supported on API 31+, by some device and HDR format combinations.
    *
    * <p>If not supported, {@link Transformer} may fall back to {@link
-   * #HDR_MODE_TONE_MAP_HDR_TO_SDR}.
+   * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}.
    */
   public static final int HDR_MODE_KEEP_HDR = 0;
   /**
-   * Tone map HDR input to SDR before processing, to generate SDR output.
+   * Tone map HDR input to SDR before processing, to generate SDR output, using the {@link
+   * android.media.MediaCodec} decoder tone-mapper.
    *
    * <p>Supported on API 31+, by some device and HDR format combinations. Tone-mapping is only
    * guaranteed to be supported from Android T onwards.
    *
-   * <p>If not supported, {@link Transformer} may throw a {@link TransformationException}.
+   * <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
    */
-  public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR = 1;
+  public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC = 1;
   /**
-   * Interpret HDR input as SDR, resulting in washed out video.
+   * Tone map HDR input to SDR before processing, to generate SDR output, using an OpenGL
+   * tone-mapper.
+   *
+   * <p>Supported on API 29+, for HLG input.
+   *
+   * <p>This may exhibit mild differences from {@link
+   * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, depending on the device's tone-mapping
+   * implementation, but should have much wider support and have more consistent results across
+   * devices.
+   *
+   * <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
+   */
+  // TODO(b/239735341): Implement PQ tone-mapping to remove the HLG reference.
+  public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL = 2;
+  /**
+   * Interpret HDR input as SDR, likely with a washed out look.
    *
    * <p>Supported on API 29+.
    *
    * <p>This is much more widely supported than {@link #HDR_MODE_KEEP_HDR} and {@link
-   * #HDR_MODE_TONE_MAP_HDR_TO_SDR}. However, as HDR transfer functions and metadata will be
-   * ignored, contents will be displayed incorrectly, likely with a washed out look.
+   * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}. However, as HDR transfer functions and
+   * metadata will be ignored, contents will be displayed incorrectly, likely with a washed out
+   * look.
    *
    * <p>Use of this flag may result in {@code
    * TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED} or {@code
@@ -85,7 +104,7 @@ public final class TransformationRequest {
    *
    * <p>This field is experimental, and will be renamed or removed in a future release.
    */
-  public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 2;
+  public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 3;
 
   /** A builder for {@link TransformationRequest} instances. */
   public static final class Builder {
@@ -283,14 +302,14 @@ public final class TransformationRequest {
 
     /**
      * @deprecated This method is now a no-op if {@code false}, and sets {@code
-     *     setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR)} if {@code true}. Use {@link #setHdrMode} with
-     *     {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR} instead.
+     *     setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)} if {@code true}. Use {@link
+     *     #setHdrMode} with {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} instead.
      */
     @Deprecated
     @CanIgnoreReturnValue
     public Builder setEnableRequestSdrToneMapping(boolean enableRequestSdrToneMapping) {
       if (enableRequestSdrToneMapping) {
-        return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR);
+        return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
       }
       return this;
     }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -16,6 +16,10 @@
 
 package com.google.android.exoplayer2.transformer;
 
+import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
+import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_KEEP_HDR;
+import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC;
+import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Util.SDK_INT;
@@ -88,9 +92,10 @@ import org.checkerframework.dataflow.qual.Pure;
       DebugViewProvider debugViewProvider)
       throws TransformationException {
     super(inputFormat, streamStartPositionUs, muxerWrapper);
+
+    boolean isGlToneMapping = false;
     if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) {
-      if (transformationRequest.hdrMode
-          == TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) {
+      if (transformationRequest.hdrMode == HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) {
         if (SDK_INT < 29) {
           throw TransformationException.createForCodec(
               new IllegalArgumentException("Interpreting HDR video as SDR is not supported."),
@@ -101,6 +106,18 @@ import org.checkerframework.dataflow.qual.Pure;
               TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED);
         }
         inputFormat = inputFormat.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
+      } else if (transformationRequest.hdrMode == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) {
+        if (SDK_INT < 29) {
+          throw TransformationException.createForCodec(
+              new IllegalArgumentException(
+                  "OpenGL-based HDR to SDR tone mapping is not supported."),
+              /* isVideo= */ true,
+              /* isDecoder= */ true,
+              inputFormat,
+              /* mediaCodecName= */ null,
+              TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED);
+        }
+        isGlToneMapping = true;
       } else if (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround()) {
         throw TransformationException.createForCodec(
             new IllegalArgumentException("HDR editing and tone mapping is not supported."),
@@ -149,19 +166,30 @@ import org.checkerframework.dataflow.qual.Pure;
             transformationRequest,
             fallbackListener);
 
-    // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
-    // This implies that the OpenGL EXT_YUV_target extension is supported and hence the
-    // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
-    // mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
-    ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor();
+    ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor();
+    // If not tone mapping using OpenGL, the decoder will output the encoderInputColor,
+    // possibly by tone mapping.
+    ColorInfo frameProcessorInputColor =
+        isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor;
+    // For consistency with the Android platform, OpenGL tone mapping outputs colors with
+    // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as
+    // C.COLOR_TRANSFER_SDR to the encoder.
+    ColorInfo frameProcessorOutputColor =
+        isGlToneMapping
+            ? new ColorInfo(
+                C.COLOR_SPACE_BT709,
+                C.COLOR_RANGE_LIMITED,
+                C.COLOR_TRANSFER_GAMMA_2_2,
+                /* hdrStaticInfo= */ null)
+            : encoderInputColor;
     try {
       frameProcessor =
           frameProcessorFactory.create(
               context,
               effectsListBuilder.build(),
               debugViewProvider,
-              /* inputColorInfo= */ encoderSupportedInputColor,
-              /* outputColorInfo= */ encoderSupportedInputColor,
+              frameProcessorInputColor,
+              frameProcessorOutputColor,
               /* releaseFramesAutomatically= */ true,
               MoreExecutors.directExecutor(),
               new FrameProcessor.Listener() {
@@ -209,12 +237,12 @@ import org.checkerframework.dataflow.qual.Pure;
         new FrameInfo(
             decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
 
-    boolean isToneMappingRequired =
+    boolean isDecoderToneMappingRequired =
         ColorInfo.isTransferHdr(inputFormat.colorInfo)
-            && !ColorInfo.isTransferHdr(encoderWrapper.getSupportedInputColor());
+            && !ColorInfo.isTransferHdr(frameProcessorInputColor);
     decoder =
         decoderFactory.createForVideoDecoding(
-            inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
+            inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired);
     maxPendingFrameCount = decoder.getMaxPendingFrameCount();
   }
 
@@ -432,7 +460,7 @@ import org.checkerframework.dataflow.qual.Pure;
     /** Returns the {@link ColorInfo} expected from the input surface. */
     public ColorInfo getSupportedInputColor() {
       boolean isHdrEditingEnabled =
-          transformationRequest.hdrMode == TransformationRequest.HDR_MODE_KEEP_HDR
+          transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
               && !supportedEncoderNamesForHdrEditing.isEmpty();
       boolean isInputToneMapped =
           !isHdrEditingEnabled && ColorInfo.isTransferHdr(inputFormat.colorInfo);
@@ -504,10 +532,12 @@ import org.checkerframework.dataflow.qual.Pure;
       boolean isInputToneMapped =
           ColorInfo.isTransferHdr(inputFormat.colorInfo)
               && !ColorInfo.isTransferHdr(requestedEncoderFormat.colorInfo);
+      // HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to
+      // HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC.
       @TransformationRequest.HdrMode
-      int hdrMode =
-          isInputToneMapped
-              ? TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR
+      int supportedFallbackHdrMode =
+          isInputToneMapped && transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
+              ? HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC
               : transformationRequest.hdrMode;
 
       fallbackListener.onTransformationRequestFinalized(
@@ -516,7 +546,7 @@ import org.checkerframework.dataflow.qual.Pure;
               /* hasOutputFormatRotation= */ flipOrientation,
               requestedEncoderFormat,
               encoderSupportedFormat,
-              hdrMode));
+              supportedFallbackHdrMode));
 
       encoderSurfaceInfo =
           new SurfaceInfo(