Add ContrastProcessor for contrast adjustments.

PiperOrigin-RevId: 462232813

demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java

@@ -104,6 +104,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
     "3D spin",
     "Overlay logo & timer",
     "Zoom in start",
+    "Increase contrast"
   };
   private static final int PERIODIC_VIGNETTE_INDEX = 2;
   private static final String SAME_AS_INPUT_OPTION = "same as input";

demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java

@@ -38,6 +38,7 @@ import androidx.appcompat.app.AppCompatActivity;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.ExoPlayer;
 import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.transformer.Contrast;
 import com.google.android.exoplayer2.transformer.DebugViewProvider;
 import com.google.android.exoplayer2.transformer.DefaultEncoderFactory;
 import com.google.android.exoplayer2.transformer.GlEffect;
@@ -321,6 +322,10 @@ public final class TransformerActivity extends AppCompatActivity {
         if (selectedEffects[5]) {
           effects.add(MatrixTransformationFactory.createZoomInTransition());
         }
+        if (selectedEffects[6]) {
+          // TODO(b/238630175): Add slider for contrast adjustments.
+          effects.add(new Contrast(0.75f));
+        }
         transformerBuilder.setVideoEffects(effects.build());
       }
 

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java

@@ -37,6 +37,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 
 /**
  * Utilities for instrumentation tests for the {@link GlEffectsFrameProcessor} and {@link
@@ -103,6 +104,17 @@ public class BitmapTestUtil {
   }
 
   /**
+   * Returns a solid {@link Bitmap} with every pixel having the same color.
+   *
+   * @param color An RGBA color created by {@link Color}.
+   */
+  public static Bitmap createArgb8888BitmapWithSolidColor(int width, int height, int color) {
+    int[] colors = new int[width * height];
+    Arrays.fill(colors, color);
+    return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
+  }
+
+  /**
    * Returns the average difference between the expected and actual bitmaps, calculated using the
    * maximum difference across all color channels for each pixel, then divided by the total number
    * of pixels in the image. The bitmap resolutions must match and they must use configuration

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/ContrastProcessorPixelTest.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Color;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import android.util.Size;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.util.GlUtil;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Pixel test for contrast adjustment via {@link ContrastProcessor}.
+ *
+ * <p>Expected images are taken from an emulator, so tests on different emulators or physical
+ * devices may fail. To test on other devices, please increase the {@link
+ * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
+ * as recommended in {@link GlEffectsFrameProcessorPixelTest}.
+ */
+@RunWith(AndroidJUnit4.class)
+public class ContrastProcessorPixelTest {
+  private static final String EXOPLAYER_LOGO_PNG_ASSET_PATH =
+      "media/bitmap/exoplayer_logo/original.png";
+  // TODO(b/239005261): Migrate png to an emulator generated picture.
+  private static final String MAXIMUM_CONTRAST_PNG_ASSET_PATH =
+      "media/bitmap/exoplayer_logo/maximum_contrast.png";
+
+  // OpenGL uses floats in [0, 1] and maps 0.5f to 128 = 256 / 2.
+  private static final int OPENGL_NEUTRAL_RGB_VALUE = 128;
+
+  private final Context context = getApplicationContext();
+
+  private @MonotonicNonNull EGLDisplay eglDisplay;
+  private @MonotonicNonNull EGLContext eglContext;
+  private @MonotonicNonNull EGLSurface placeholderEglSurface;
+  private @MonotonicNonNull SingleFrameGlTextureProcessor contrastProcessor;
+  private int inputTexId;
+  private int outputTexId;
+  private int inputWidth;
+  private int inputHeight;
+
+  @Before
+  public void createGlObjects() throws Exception {
+    eglDisplay = GlUtil.createEglDisplay();
+    eglContext = GlUtil.createEglContext(eglDisplay);
+
+    Bitmap inputBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH);
+    inputWidth = inputBitmap.getWidth();
+    inputHeight = inputBitmap.getHeight();
+
+    placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay);
+    GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, inputWidth, inputHeight);
+    inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap);
+  }
+
+  @After
+  public void release() throws GlUtil.GlException, FrameProcessingException {
+    if (contrastProcessor != null) {
+      contrastProcessor.release();
+    }
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+  }
+
+  @Test
+  public void drawFrame_noContrastChange_leavesFrameUnchanged() throws Exception {
+    String testId = "drawFrame_noContrastChange";
+    contrastProcessor =
+        new Contrast(/* contrast= */ 0.0f).toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = contrastProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH);
+
+    contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.getWidth(), outputSize.getHeight());
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_minimumContrast_producesAllGrayFrame() throws Exception {
+    String testId = "drawFrame_minimumContrast";
+    contrastProcessor =
+        new Contrast(/* contrast= */ -1.0f).toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = contrastProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap expectedBitmap =
+        BitmapTestUtil.createArgb8888BitmapWithSolidColor(
+            inputWidth,
+            inputHeight,
+            Color.rgb(
+                OPENGL_NEUTRAL_RGB_VALUE, OPENGL_NEUTRAL_RGB_VALUE, OPENGL_NEUTRAL_RGB_VALUE));
+
+    contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.getWidth(), outputSize.getHeight());
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_decreaseContrast_decreasesPixelsGreaterEqual128IncreasesBelow()
+      throws Exception {
+    String testId = "drawFrame_decreaseContrast";
+    contrastProcessor =
+        new Contrast(/* contrast= */ -0.75f).toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = contrastProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap originalBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH);
+
+    contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.getWidth(), outputSize.getHeight());
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap);
+    assertIncreasedOrDecreasedContrast(originalBitmap, actualBitmap, /* increased= */ false);
+  }
+
+  @Test
+  public void drawFrame_increaseContrast_increasesPixelsGreaterEqual128DecreasesBelow()
+      throws Exception {
+    String testId = "drawFrame_increaseContrast";
+    contrastProcessor =
+        new Contrast(/* contrast= */ 0.75f).toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = contrastProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap originalBitmap = BitmapTestUtil.readBitmap(EXOPLAYER_LOGO_PNG_ASSET_PATH);
+
+    contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.getWidth(), outputSize.getHeight());
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap);
+    assertIncreasedOrDecreasedContrast(originalBitmap, actualBitmap, /* increased= */ true);
+  }
+
+  @Test
+  public void drawFrame_maximumContrast_pixelEither0or255() throws Exception {
+    String testId = "drawFrame_maximumContrast";
+    contrastProcessor =
+        new Contrast(/* contrast= */ 1.0f).toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = contrastProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap expectedBitmap = BitmapTestUtil.readBitmap(MAXIMUM_CONTRAST_PNG_ASSET_PATH);
+
+    contrastProcessor.drawFrame(inputTexId, /* presentationTimeUs = */ 0);
+    Bitmap actualBitmap =
+        BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+            outputSize.getWidth(), outputSize.getHeight());
+
+    BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(testId, "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
+            expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  private static void assertIncreasedOrDecreasedContrast(
+      Bitmap originalBitmap, Bitmap actualBitmap, boolean increased) {
+
+    for (int y = 0; y < actualBitmap.getHeight(); y++) {
+      for (int x = 0; x < actualBitmap.getWidth(); x++) {
+        int originalColor = originalBitmap.getPixel(x, y);
+        int actualColor = actualBitmap.getPixel(x, y);
+
+        int redDifference = Color.red(actualColor) - Color.red(originalColor);
+        int greenDifference = Color.green(actualColor) - Color.green(originalColor);
+        int blueDifference = Color.blue(actualColor) - Color.blue(originalColor);
+
+        // If the contrast increases, all pixels with a value greater or equal to
+        // OPENGL_NEUTRAL_RGB_VALUE must increase (diff is greater or equal to 0) and all pixels
+        // below OPENGL_NEUTRAL_RGB_VALUE must decrease (diff is smaller or equal to 0).
+        // If the contrast decreases, all pixels with a value greater or equal to
+        // OPENGL_NEUTRAL_RGB_VALUE must decrease (diff is smaller or equal to 0) and all pixels
+        // below OPENGL_NEUTRAL_RGB_VALUE must increase (diff is greater or equal to 0).
+        // The interval limits 0 and 255 stay unchanged for either contrast in- or decrease.
+
+        if (Color.red(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) {
+          assertThat(increased ? redDifference : -redDifference).isAtLeast(0);
+        } else {
+          assertThat(increased ? redDifference : -redDifference).isAtMost(0);
+        }
+
+        if (Color.green(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) {
+          assertThat(increased ? greenDifference : -greenDifference).isAtLeast(0);
+        } else {
+          assertThat(increased ? greenDifference : -greenDifference).isAtMost(0);
+        }
+
+        if (Color.blue(originalColor) >= OPENGL_NEUTRAL_RGB_VALUE) {
+          assertThat(increased ? blueDifference : -blueDifference).isAtLeast(0);
+        } else {
+          assertThat(increased ? blueDifference : -blueDifference).isAtMost(0);
+        }
+      }
+    }
+  }
+
+  private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
+    outputTexId =
+        GlUtil.createTexture(
+            outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
+    int frameBuffer = GlUtil.createFboForTexture(outputTexId);
+    GlUtil.focusFramebuffer(
+        checkNotNull(eglDisplay),
+        checkNotNull(eglContext),
+        checkNotNull(placeholderEglSurface),
+        frameBuffer,
+        outputWidth,
+        outputHeight);
+  }
+}

library/transformer/src/main/assets/shaders/fragment_shader_contrast_es2.glsl

+#version 100
+// Copyright 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 2 fragment shader that samples from a (non-external) texture with
+// uTexSampler, copying from this texture to the current output
+// while adjusting contrast based on uContrastFactor.
+
+precision mediump float;
+uniform sampler2D uTexSampler;
+uniform float uContrastFactor;
+varying vec2 vTexSamplingCoord;
+
+void main() {
+    vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
+
+    gl_FragColor = vec4(
+        clamp(uContrastFactor * (inputColor.r - 0.5) + 0.5, 0.0, 1.0),
+        clamp(uContrastFactor * (inputColor.g - 0.5) + 0.5, 0.0, 1.0),
+        clamp(uContrastFactor * (inputColor.b - 0.5) + 0.5, 0.0, 1.0),
+        inputColor.a);
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Contrast.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+
+import android.content.Context;
+
+/** A {@link GlEffect} to control the contrast of video frames. */
+public class Contrast implements GlEffect {
+
+  /** Adjusts the contrast of video frames in the interval [-1, 1]. */
+  public final float contrast;
+
+  /**
+   * Creates a new instance for the given contrast value.
+   *
+   * <p>Contrast values range from -1 (all gray pixels) to 1 (maximum difference of colors). 0 means
+   * to add no contrast and leaves the frames unchanged.
+   */
+  public Contrast(float contrast) {
+    checkArgument(-1 <= contrast && contrast <= 1, "Contrast needs to be in the interval [-1, 1].");
+    this.contrast = contrast;
+  }
+
+  @Override
+  public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
+      throws FrameProcessingException {
+    return new ContrastProcessor(context, this, useHdr);
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ContrastProcessor.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import android.content.Context;
+import android.opengl.GLES20;
+import android.opengl.Matrix;
+import android.util.Size;
+import com.google.android.exoplayer2.util.GlProgram;
+import com.google.android.exoplayer2.util.GlUtil;
+import java.io.IOException;
+
+/** Contrast processor to apply a {@link Contrast} to each frame. */
+/* package */ final class ContrastProcessor extends SingleFrameGlTextureProcessor {
+  private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_contrast_es2.glsl";
+
+  private final GlProgram glProgram;
+  private final float contrastFactor;
+
+  public ContrastProcessor(Context context, Contrast contrastEffect, boolean useHdr)
+      throws FrameProcessingException {
+    super(useHdr);
+    // Use 1.0001f to avoid division by zero issues.
+    contrastFactor = (1 + contrastEffect.contrast) / (1.0001f - contrastEffect.contrast);
+
+    try {
+      glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
+    } catch (IOException | GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+
+    // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
+    glProgram.setBufferAttribute(
+        "aFramePosition",
+        GlUtil.getNormalizedCoordinateBounds(),
+        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+
+    float[] identityMatrix = new float[16];
+    Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
+    glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
+    glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
+  }
+
+  @Override
+  public Size configure(int inputWidth, int inputHeight) {
+    return new Size(inputWidth, inputHeight);
+  }
+
+  @Override
+  public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
+    try {
+      glProgram.use();
+      glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
+      glProgram.setFloatUniform("uContrastFactor", contrastFactor);
+      glProgram.bindAttributesAndUniforms();
+
+      // The four-vertex triangle strip forms a quad.
+      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e, presentationTimeUs);
+    }
+  }
+}

testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/README.md → testdata/src/test/assets/media/bitmap/README.md

-Expected first frame of
-[sample.mp4](https://github.com/androidx/media/blob/main/libraries/test_data/src/test/assets/media/mp4/sample.mp4)
-after a
+Expected first frame after a
 [Transformer](https://github.com/androidx/media/tree/main/libraries/transformer)
 transformation. Used to validate that frame operations produce expected output
 in