Add LUT functionalities to transformer.

* Adds SDR 3D LUT functionalities with OpenGL 2.0 support.

PiperOrigin-RevId: 474561060
(cherry picked from commit 702419dd)

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/assets/shaders/fragment_shader_lut_es2.glsl

+#version 100
+// Copyright 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES2 fragment shader that samples from a (non-external) texture with
+// uTexSampler, copying from this texture to the current output while
+// applying a 3D color lookup table to change the pixel colors.
+
+precision highp float;
+uniform sampler2D uTexSampler;
+// The uColorLut texture is a N x N^2 2D texture where each z-plane of the 3D
+// LUT is vertically stacked on top of each other. The red channel of the input
+// color (z-axis in LUT[R][G][B] = LUT[z][y][x]) points to the plane to sample
+// from. For more information check the
+// androidx/media3/effect/SingleColorLut.java class, especially the function
+// #transformCubeIntoBitmap with a provided example.
+uniform sampler2D uColorLut;
+uniform float uColorLutLength;
+varying vec2 vTexSamplingCoord;
+
+// Applies the color lookup using uLut based on the input colors.
+vec3 applyLookup(vec3 color) {
+    // Reminder: Inside OpenGL vector.xyz is the same as vector.rgb.
+    // Here we use mentions of x and y coordinates to references to
+    // the position to sample from inside the 2D LUT plane and
+    // rgb to create the 3D coordinates based on the input colors.
+
+    // To sample from the 3D LUT we interpolate bilinearly twice in the 2D LUT
+    // to replicate the trilinear interpolation in a 3D LUT. Thus we sample
+    // from the plane of position redCoordLow and on the plane above.
+    // redCoordLow points to the lower plane to sample from.
+    float redCoord = color.r * (uColorLutLength - 1.0);
+    // Clamping to uColorLutLength - 2 is only needed if redCoord points to the
+    // most upper plane. In this case there would not be any plane above
+    // available to sample from.
+    float redCoordLow = clamp(floor(redCoord), 0.0, uColorLutLength - 2.0);
+
+    // lowerY is indexed in two steps. First redCoordLow defines the plane to
+    // sample from. Next the green color component is added to index the row in
+    // the found plane. As described in the NVIDIA blog article about LUTs
+    // https://developer.nvidia.com/gpugems/gpugems2/part-iii-high-quality-rendering/chapter-24-using-lookup-tables-accelerate-color
+    // (Section 24.2), we sample from color * scale + offset, where offset is
+    // defined by 1 / (2 * uColorLutLength) and the scale is defined by
+    // (uColorLutLength - 1.0) / uColorLutLength.
+
+    // The following derives the equation of lowerY. For this let
+    // N = uColorLutLenght. The general formula to sample at row y
+    // is defined as y = N * r + g.
+    // Using the offset and scale as described in NVIDIA's blog article we get:
+    // y = offset + (N * r + g) * scale
+    // y = 1 / (2 * N) + (N * r + g) * (N - 1) / N
+    // y = 1 / (2 * N) + N * r * (N - 1) / N + g * (N - 1) / N
+    // We have defined redCoord as r * (N - 1) if we excluded the clamping for
+    // now, giving us:
+    // y = 1 / (2 * N) + N * redCoord / N + g * (N - 1) / N
+    // This simplifies to:
+    // y = 0.5 / N + (N * redCoord + g * (N - 1)) / N
+    // y = (0.5 + N * redCoord + g * (N - 1)) / N
+    // This formula now assumes a coordinate system in the range of [0, N] but
+    // OpenGL uses a [0, 1] unit coordinate system internally. Thus dividing
+    // by N gives us the final formula for y:
+    // y = ((0.5 + N * redCoord + g * (N - 1)) / N) / N
+    // y = (0.5 + redCoord * N + g * (N - 1)) / (N * N)
+    float lowerY =
+        (0.5
+            + redCoordLow * uColorLutLength
+            + color.g * (uColorLutLength - 1.0))
+        / (uColorLutLength * uColorLutLength);
+    // The upperY is the same position moved up by one LUT plane.
+    float upperY = lowerY + 1.0 / uColorLutLength;
+
+    // The x position is the blue color channel (x-axis in LUT[R][G][B]).
+    float x = (0.5 + color.b * (uColorLutLength - 1.0)) / uColorLutLength;
+
+    vec3 lowerRgb = texture2D(uColorLut, vec2(x, lowerY)).rgb;
+    vec3 upperRgb = texture2D(uColorLut, vec2(x, upperY)).rgb;
+
+    // Linearly interpolate between lowerRgb and upperRgb based on the
+    // distance of the actual in the plane and the lower sampling position.
+    return mix(lowerRgb, upperRgb, redCoord - redCoordLow);
+}
+
+void main() {
+    vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
+
+    gl_FragColor.rgb = applyLookup(inputColor.rgb);
+    gl_FragColor.a = inputColor.a;
+}

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ColorLut.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import android.content.Context;
+import androidx.media3.common.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlUtil;
+
+/**
+ * Specifies color transformations using color lookup tables to apply to each frame in the fragment
+ * shader.
+ */
+public interface ColorLut extends GlEffect {
+
+  /**
+   * Returns the OpenGL texture ID of the LUT to apply to the pixels of the frame with the given
+   * timestamp.
+   */
+  int getLutTextureId(long presentationTimeUs);
+
+  /** Returns the length N of the 3D N x N x N LUT cube with the given timestamp. */
+  int getLength(long presentationTimeUs);
+
+  /** Releases the OpenGL texture of the LUT. */
+  void release() throws GlUtil.GlException;
+
+  @Override
+  default ColorLutProcessor toGlTextureProcessor(Context context, boolean useHdr)
+      throws FrameProcessingException {
+    return new ColorLutProcessor(context, /* colorLut= */ this, useHdr);
+  }
+}

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ColorLutProcessor.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+
+import android.content.Context;
+import android.opengl.GLES20;
+import android.opengl.Matrix;
+import android.util.Pair;
+import androidx.media3.common.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlProgram;
+import com.google.android.exoplayer2.util.GlUtil;
+import java.io.IOException;
+
+/** Applies a {@link ColorLut} to each frame in the fragment shader. */
+/* package */ final class ColorLutProcessor extends SingleFrameGlTextureProcessor {
+  private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_lut_es2.glsl";
+
+  private final GlProgram glProgram;
+  private final ColorLut colorLut;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param colorLut The {@link ColorLut} to apply to each frame in order.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   * @throws FrameProcessingException If a problem occurs while reading shader files.
+   */
+  public ColorLutProcessor(Context context, ColorLut colorLut, boolean useHdr)
+      throws FrameProcessingException {
+    super(useHdr);
+    // TODO(b/246315245): Add HDR support.
+    checkArgument(!useHdr, "LutProcessor does not support HDR colors.");
+    this.colorLut = colorLut;
+
+    try {
+      glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
+    } catch (IOException | GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+
+    // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
+    glProgram.setBufferAttribute(
+        "aFramePosition",
+        GlUtil.getNormalizedCoordinateBounds(),
+        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+
+    float[] identityMatrix = new float[16];
+    Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
+    glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
+    glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
+  }
+
+  @Override
+  public Pair<Integer, Integer> configure(int inputWidth, int inputHeight) {
+    return Pair.create(inputWidth, inputHeight);
+  }
+
+  @Override
+  public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
+    try {
+      glProgram.use();
+      glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
+      glProgram.setSamplerTexIdUniform(
+          "uColorLut", colorLut.getLutTextureId(presentationTimeUs), /* texUnitIndex= */ 1);
+      glProgram.setFloatUniform("uColorLutLength", colorLut.getLength(presentationTimeUs));
+      glProgram.bindAttributesAndUniforms();
+
+      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+  }
+
+  @Override
+  public void release() throws FrameProcessingException {
+    super.release();
+    try {
+      colorLut.release();
+      glProgram.delete();
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+  }
+}

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/SingleColorLut.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+import static com.google.android.exoplayer2.util.Assertions.checkState;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.opengl.GLES20;
+import android.opengl.GLUtils;
+import androidx.media3.common.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlUtil;
+import com.google.android.exoplayer2.util.Util;
+
+/** Transforms the colors of a frame by applying the same color lookup table to each frame. */
+public class SingleColorLut implements ColorLut {
+  private final int lutTextureId;
+  private final int length;
+
+  /**
+   * Creates a new instance.
+   *
+   * <p>{@code lutCube} needs to be a {@code N x N x N} cube and each element is an integer
+   * representing a color using the {@link Bitmap.Config#ARGB_8888} format.
+   */
+  public static SingleColorLut createFromCube(int[][][] lutCube) throws GlUtil.GlException {
+    checkArgument(
+        lutCube.length > 0 && lutCube[0].length > 0 && lutCube[0][0].length > 0,
+        "LUT must have three dimensions.");
+    checkArgument(
+        lutCube.length == lutCube[0].length && lutCube.length == lutCube[0][0].length,
+        Util.formatInvariant(
+            "All three dimensions of a LUT must match, received %d x %d x %d.",
+            lutCube.length, lutCube[0].length, lutCube[0][0].length));
+
+    return new SingleColorLut(transformCubeIntoBitmap(lutCube));
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * <p>LUT needs to be a Bitmap of a flattened HALD image of width {@code N} and height {@code
+   * N^2}. Each element must be an integer representing a color using the {@link
+   * Bitmap.Config#ARGB_8888} format.
+   */
+  public static SingleColorLut createFromBitmap(Bitmap lut) throws GlUtil.GlException {
+    checkArgument(
+        lut.getWidth() * lut.getWidth() == lut.getHeight(),
+        Util.formatInvariant(
+            "LUT needs to be in a N x N^2 format, received %d x %d.",
+            lut.getWidth(), lut.getHeight()));
+    checkArgument(
+        lut.getConfig() == Bitmap.Config.ARGB_8888, "Color representation needs to be ARGB_8888.");
+
+    return new SingleColorLut(lut);
+  }
+
+  private SingleColorLut(Bitmap lut) throws GlUtil.GlException {
+    length = lut.getWidth();
+    lutTextureId = storeLutAsTexture(lut);
+  }
+
+  private static int storeLutAsTexture(Bitmap bitmap) throws GlUtil.GlException {
+    int lutTextureId =
+        GlUtil.createTexture(
+            bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
+    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
+    GlUtil.checkGlError();
+    return lutTextureId;
+  }
+
+  /**
+   * Transforms the N x N x N {@code cube} into a N x N^2 {@code bitmap}.
+   *
+   * @param cube The 3D Color Lut which gets indexed using {@code cube[R][G][B]}.
+   * @return A {@link Bitmap} of size {@code N x N^2}, where the {@code cube[R][G][B]} color can be
+   *     indexed at {@code bitmap.getColor(B, N * R + G)}.
+   */
+  private static Bitmap transformCubeIntoBitmap(int[][][] cube) {
+    // The support for 3D textures starts in OpenGL 3.0 and the Android API 8, Version 2.2
+    // uses OpenGL 2.0 which only supports 2D textures. Thus we need to transform the 3D LUT
+    // into 2D to support all Android SDKs.
+
+    // The cube consists of N planes on the z-direction in the coordinate system where each plane
+    // has a size of N x N. To transform the cube into a 2D bitmap we stack each N x N plane
+    // vertically on top of each other. This gives us a bitmap of width N and height N^2.
+    //
+    // As an example, lets take the following 3D identity LUT of size 2x2x2:
+    // cube = [
+    //         [[(0, 0, 0), (0, 0, 1)],
+    //          [(0, 1, 0), (0, 1, 1)]],
+    //         [[(1, 0, 0), (1, 0, 1)],
+    //          [(1, 1, 0), (1, 1, 1)]]
+    //       ];
+    // If we transform this cube now into a 2x2^2 = 2x4 bitmap we yield the following 2D plane:
+    // bitmap = [[(0, 0, 0), (0, 0, 1)],
+    //           [(0, 1, 0), (0, 1, 1)],
+    //           [(1, 0, 0), (1, 0, 1)],
+    //           [(1, 1, 0), (1, 1, 1)]];
+    // media/bitmap/lut/identity.png is an example of how a 32x32x32 3D LUT looks like as an
+    // 32x32^2 bitmap.
+    int length = cube.length;
+    int[] bitmapColorsArray = new int[length * length * length];
+
+    for (int r = 0; r < length; r++) {
+      for (int g = 0; g < length; g++) {
+        for (int b = 0; b < length; b++) {
+          int color = cube[r][g][b];
+          int planePosition = b + length * (g + length * r);
+          bitmapColorsArray[planePosition] = color;
+        }
+      }
+    }
+
+    return Bitmap.createBitmap(
+        bitmapColorsArray,
+        /* width= */ length,
+        /* height= */ length * length,
+        Bitmap.Config.ARGB_8888);
+  }
+
+  @Override
+  public int getLutTextureId(long presentationTimeUs) {
+    return lutTextureId;
+  }
+
+  @Override
+  public int getLength(long presentationTimeUs) {
+    return length;
+  }
+
+  @Override
+  public void release() throws GlUtil.GlException {
+    GlUtil.deleteTexture(lutTextureId);
+  }
+
+  @Override
+  public ColorLutProcessor toGlTextureProcessor(Context context, boolean useHdr)
+      throws FrameProcessingException {
+    checkState(!useHdr, "HDR is currently not supported.");
+    return new ColorLutProcessor(context, /* colorLut= */ this, useHdr);
+  }
+}