Add the processor, GLEffect, texture and bitmap overlays

Implements milestone 1.1 of the [overlays implementation plan](https://docs.google.com/document/d/1EcP2GN8k8N74hHZyD0KTqm9oQo5-W1dZMqIVyqVGtlo/edit#bookmark=id.76uzcie1dg9d)

PiperOrigin-RevId: 491696361

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/BitmapTestUtil.java

@@ -24,7 +24,6 @@ import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.Color;
-import android.graphics.Matrix;
 import android.graphics.PixelFormat;
 import android.media.Image;
 import android.opengl.GLES20;
@@ -211,7 +210,7 @@ public class BitmapTestUtil {
     // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888.
     bitmap.copyPixelsFromBuffer(rgba8888Buffer);
     // Flip the bitmap as its positive y-axis points down while OpenGL's positive y-axis points up.
-    return flipBitmapVertically(bitmap);
+    return BitmapUtil.flipBitmapVertically(bitmap);
   }
 
   /**
@@ -227,23 +226,10 @@ public class BitmapTestUtil {
             bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
     // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down
     // while OpenGL's positive y-axis points up.
-    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0);
+    GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, BitmapUtil.flipBitmapVertically(bitmap), 0);
     GlUtil.checkGlError();
     return texId;
   }
 
-  private static Bitmap flipBitmapVertically(Bitmap bitmap) {
-    Matrix flip = new Matrix();
-    flip.postScale(1f, -1f);
-    return Bitmap.createBitmap(
-        bitmap,
-        /* x= */ 0,
-        /* y= */ 0,
-        bitmap.getWidth(),
-        bitmap.getHeight(),
-        flip,
-        /* filter= */ true);
-  }
-
   private BitmapTestUtil() {}
 }

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.createGlTextureFromBitmap;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory;
+import static com.google.android.exoplayer2.effect.BitmapTestUtil.readBitmap;
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLSurface;
+import android.opengl.Matrix;
+import android.util.Pair;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.util.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlUtil;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Pixel test for texture processing via {@link OverlayTextureProcessor}.
+ *
+ * <p>Expected bitmaps are taken from an emulator, so tests on different emulators or physical
+ * devices may fail. To test on other devices, please increase the {@link
+ * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
+ * as recommended in {@link GlEffectsFrameProcessorPixelTest}.
+ */
+@RunWith(AndroidJUnit4.class)
+public class OverlayTextureProcessorPixelTest {
+  public static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/overlay/100winners.png";
+  public static final String ORIGINAL_PNG_ASSET_PATH =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
+  public static final String OVERLAY_BITMAP_DEFAULT =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png";
+  public static final String OVERLAY_BITMAP_SCALED =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png";
+
+  private final Context context = getApplicationContext();
+
+  private @MonotonicNonNull EGLDisplay eglDisplay;
+  private @MonotonicNonNull EGLContext eglContext;
+  private @MonotonicNonNull SingleFrameGlTextureProcessor overlayTextureProcessor;
+  private @MonotonicNonNull EGLSurface placeholderEglSurface;
+  private int inputTexId;
+  private int inputWidth;
+  private int inputHeight;
+
+  @Before
+  public void createGlObjects() throws IOException, GlUtil.GlException {
+    eglDisplay = GlUtil.createEglDisplay();
+    eglContext = GlUtil.createEglContext(eglDisplay);
+    placeholderEglSurface = GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
+
+    Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+    inputWidth = inputBitmap.getWidth();
+    inputHeight = inputBitmap.getHeight();
+    inputTexId = createGlTextureFromBitmap(inputBitmap);
+  }
+
+  @After
+  public void release() throws GlUtil.GlException, FrameProcessingException {
+    if (overlayTextureProcessor != null) {
+      overlayTextureProcessor.release();
+    }
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+  }
+
+  @Test
+  public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception {
+    String testId = "drawFrame_noOverlays";
+    overlayTextureProcessor =
+        new OverlayEffect(/* textureOverlays= */ ImmutableList.of())
+            .toGlTextureProcessor(context, /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.first, outputSize.second);
+    Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+
+    overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second);
+
+    maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_bitmapOverlay_blendsBitmapIntoFrame() throws Exception {
+    String testId = "drawFrame_bitmapOverlay";
+    Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
+    BitmapOverlay scaledBitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap);
+    overlayTextureProcessor =
+        new OverlayEffect(ImmutableList.of(scaledBitmapOverlay))
+            .toGlTextureProcessor(context, /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.first, outputSize.second);
+    Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_DEFAULT);
+
+    overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second);
+
+    maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  @Test
+  public void drawFrame_scaledBitmapOverlay_blendsBitmapIntoFrame() throws Exception {
+    String testId = "drawFrame_scaledBitmapOverlay";
+    Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
+    float[] scaleMatrix = GlUtil.create4x4IdentityMatrix();
+    Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1);
+    OverlaySettings overlaySettings = new OverlaySettings.Builder().setMatrix(scaleMatrix).build();
+    BitmapOverlay staticBitmapOverlay =
+        BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
+    overlayTextureProcessor =
+        new OverlayEffect(ImmutableList.of(staticBitmapOverlay))
+            .toGlTextureProcessor(context, /* useHdr= */ false);
+    Pair<Integer, Integer> outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.first, outputSize.second);
+    Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED);
+
+    overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second);
+
+    maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
+    int outputTexId =
+        GlUtil.createTexture(
+            outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
+    int frameBuffer = GlUtil.createFboForTexture(outputTexId);
+    GlUtil.focusFramebuffer(
+        checkNotNull(eglDisplay),
+        checkNotNull(eglContext),
+        checkNotNull(placeholderEglSurface),
+        frameBuffer,
+        outputWidth,
+        outputHeight);
+  }
+}

library/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl

+#version 100
+// Copyright 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 2 fragment shader that overlays a bitmap over a video frame.
+
+precision mediump float;
+// Texture containing an input video frame.
+uniform sampler2D uVideoTexSampler0;
+// Texture containing the overlay bitmap.
+uniform sampler2D uOverlayTexSampler1;
+varying vec2 vVideoTexSamplingCoord;
+varying vec2 vOverlayTexSamplingCoord1;
+
+// Manually implementing the CLAMP_TO_BORDER texture wrapping option
+// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.
+vec4 getClampToBorderOverlayColor() {
+  if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0
+    || vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0){
+    return vec4(0.0, 0.0, 0.0, 0.0);
+  } else {
+    return vec4(texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1));
+  }
+}
+
+float getMixAlpha(float videoAlpha, float overlayAlpha) {
+  if (videoAlpha == 0.0){
+    return 1.0;
+  } else {
+    return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);
+  }
+}
+
+void main() {
+  vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord));
+  vec4 overlayColor = getClampToBorderOverlayColor();
+
+  // Blend the video decoder output and the overlay bitmap.
+  gl_FragColor = mix(
+    videoColor, overlayColor, getMixAlpha(videoColor.a, overlayColor.a));
+}

library/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl

+#version 100
+// Copyright 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 2 vertex shader that leaves the frame coordinates unchanged
+// and applies matrix transformations to the texture coordinates.
+
+uniform mat4 uAspectRatioMatrix;
+uniform mat4 uOverlayMatrix;
+attribute vec4 aFramePosition;
+varying vec2 vVideoTexSamplingCoord;
+varying vec2 vOverlayTexSamplingCoord1;
+
+
+vec2 getTexSamplingCoord(vec2 ndcPosition) {
+  return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5);
+}
+
+void main() {
+  gl_Position = aFramePosition;
+  vec4 aOverlayPosition = uAspectRatioMatrix * uOverlayMatrix * aFramePosition;
+  vOverlayTexSamplingCoord1 = getTexSamplingCoord(aOverlayPosition.xy);
+  vVideoTexSamplingCoord = getTexSamplingCoord(aFramePosition.xy);
+}
+
+

library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapLoader.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package com.google.android.exoplayer2.effect;
+
+import android.graphics.Bitmap;
+import android.net.Uri;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.MediaMetadata;
+import com.google.common.util.concurrent.ListenableFuture;
+
+// TODO(b/258685047): delete this copy once session/BitmapLoader.java is moved to common
+//  (b/194284041, b/258658893).
+
+/** Loads images. */
+public interface BitmapLoader {
+  /** Decodes an image from compressed binary data. */
+  ListenableFuture<Bitmap> decodeBitmap(byte[] data);
+
+  /** Loads an image from {@code uri}. */
+  ListenableFuture<Bitmap> loadBitmap(Uri uri);
+
+  /**
+   * Loads an image from {@link MediaMetadata}. Returns null if {@code metadata} doesn't contain
+   * bitmap information.
+   *
+   * <p>By default, the method will try to decode an image from {@link MediaMetadata#artworkData} if
+   * it is present. Otherwise, the method will try to load an image from {@link
+   * MediaMetadata#artworkUri} if it is present. The method will return null if neither {@link
+   * MediaMetadata#artworkData} nor {@link MediaMetadata#artworkUri} is present.
+   */
+  @Nullable
+  default ListenableFuture<Bitmap> loadBitmapFromMetadata(MediaMetadata metadata) {
+    @Nullable ListenableFuture<Bitmap> future;
+    if (metadata.artworkData != null) {
+      future = decodeBitmap(metadata.artworkData);
+    } else if (metadata.artworkUri != null) {
+      future = loadBitmap(metadata.artworkUri);
+    } else {
+      future = null;
+    }
+    return future;
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapOverlay.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+
+import android.graphics.Bitmap;
+import android.net.Uri;
+import android.opengl.GLES20;
+import android.opengl.GLUtils;
+import android.util.Pair;
+import com.google.android.exoplayer2.util.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlUtil;
+import com.google.common.util.concurrent.ListenableFuture;
+import java.util.concurrent.ExecutionException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/**
+ * Creates {@link TextureOverlay}s from {@link Bitmap}s.
+ *
+ * <p>Useful for overlaying images and animated images (e.g. GIFs).
+ */
+public abstract class BitmapOverlay extends TextureOverlay {
+  private int lastTextureId;
+  private @MonotonicNonNull Bitmap lastBitmap;
+
+  /**
+   * Returns the overlay bitmap displayed at the specified timestamp.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   * @throws FrameProcessingException If an error occurs while processing or drawing the frame.
+   */
+  public abstract Bitmap getBitmap(long presentationTimeUs) throws FrameProcessingException;
+
+  /**
+   * {@inheritDoc}
+   *
+   * <p>Gets the width and height of the cached bitmap.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   */
+  @Override
+  public Pair<Integer, Integer> getTextureSize(long presentationTimeUs) {
+    return Pair.create(checkNotNull(lastBitmap).getWidth(), checkNotNull(lastBitmap).getHeight());
+  }
+
+  @Override
+  public int getTextureId(long presentationTimeUs) throws FrameProcessingException {
+    Bitmap bitmap = getBitmap(presentationTimeUs);
+    if (bitmap != lastBitmap) {
+      try {
+        lastBitmap = bitmap;
+        lastTextureId =
+            GlUtil.createTexture(
+                bitmap.getWidth(),
+                bitmap.getHeight(),
+                /* useHighPrecisionColorComponents= */ false);
+        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, lastTextureId);
+        GLUtils.texImage2D(
+            GLES20.GL_TEXTURE_2D,
+            /* level= */ 0,
+            BitmapUtil.flipBitmapVertically(lastBitmap),
+            /* border= */ 0);
+        GlUtil.checkGlError();
+      } catch (GlUtil.GlException e) {
+        throw new FrameProcessingException(e);
+      }
+    }
+    return lastTextureId;
+  }
+
+  /**
+   * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same position and
+   * size throughout the whole video.
+   */
+  public static BitmapOverlay createStaticBitmapOverlay(Bitmap overlayBitmap) {
+    return new BitmapOverlay() {
+      @Override
+      public Bitmap getBitmap(long presentationTimeUs) {
+        return overlayBitmap;
+      }
+    };
+  }
+
+  /**
+   * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same {@link
+   * OverlaySettings} throughout the whole video.
+   *
+   * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on
+   *     the frames.
+   */
+  public static BitmapOverlay createStaticBitmapOverlay(
+      Bitmap overlayBitmap, OverlaySettings overlaySettings) {
+    return new BitmapOverlay() {
+      @Override
+      public Bitmap getBitmap(long presentationTimeUs) {
+        return overlayBitmap;
+      }
+
+      @Override
+      public OverlaySettings getOverlaySettings(long presentationTimeUs) {
+        return overlaySettings;
+      }
+    };
+  }
+
+  /**
+   * Creates a {@link BitmapOverlay} that shows the input at {@code overlayBitmapUri} with the same
+   * {@link OverlaySettings} throughout the whole video.
+   *
+   * @param overlayBitmapUri The {@link Uri} pointing to the resource to be converted into a bitmap.
+   * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on
+   *     the frames.
+   */
+  public static BitmapOverlay createStaticBitmapOverlay(
+      Uri overlayBitmapUri, OverlaySettings overlaySettings) {
+    return new BitmapOverlay() {
+      private @MonotonicNonNull Bitmap lastBitmap;
+
+      @Override
+      public Bitmap getBitmap(long presentationTimeUs) throws FrameProcessingException {
+        if (lastBitmap == null) {
+          BitmapLoader bitmapLoader = new SimpleBitmapLoader();
+          ListenableFuture<Bitmap> future = bitmapLoader.loadBitmap(overlayBitmapUri);
+          try {
+            lastBitmap = future.get();
+          } catch (ExecutionException | InterruptedException e) {
+            throw new FrameProcessingException(e);
+          }
+        }
+        return lastBitmap;
+      }
+
+      @Override
+      public OverlaySettings getOverlaySettings(long presentationTimeUs) {
+        return overlaySettings;
+      }
+    };
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapUtil.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+
+/** Utility functions for working with {@link Bitmap}. */
+/* package */ final class BitmapUtil {
+  static Bitmap flipBitmapVertically(Bitmap bitmap) {
+    Matrix flip = new Matrix();
+    flip.postScale(1f, -1f);
+    return Bitmap.createBitmap(
+        bitmap,
+        /* x= */ 0,
+        /* y= */ 0,
+        bitmap.getWidth(),
+        bitmap.getHeight(),
+        flip,
+        /* filter= */ true);
+  }
+
+  /** Class only contains static methods. */
+  private BitmapUtil() {}
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayEffect.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import android.content.Context;
+import com.google.android.exoplayer2.util.FrameProcessingException;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Applies a list of {@link TextureOverlay}s to a frame in FIFO order (the last overlay in the list
+ * is displayed on top).
+ */
+public final class OverlayEffect implements GlEffect {
+
+  private final ImmutableList<TextureOverlay> overlays;
+
+  /**
+   * Creates a new instance for the given list of {@link TextureOverlay}s.
+   *
+   * @param textureOverlays The {@link TextureOverlay}s to be blended into the frame.
+   */
+  public OverlayEffect(ImmutableList<TextureOverlay> textureOverlays) {
+    this.overlays = textureOverlays;
+  }
+
+  @Override
+  public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
+      throws FrameProcessingException {
+    return new OverlayTextureProcessor(context, useHdr, overlays);
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java

+package com.google.android.exoplayer2.effect;
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.google.android.exoplayer2.util.GlUtil;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+
+/** Contains information to control how an {@link TextureOverlay} is displayed on the screen. */
+public final class OverlaySettings {
+  public final boolean useHdr;
+  public final float[] matrix;
+
+  private OverlaySettings(boolean useHdr, float[] matrix) {
+    this.useHdr = useHdr;
+    this.matrix = matrix;
+  }
+
+  /** A builder for {@link OverlaySettings} instances. */
+  public static final class Builder {
+    private boolean useHdr;
+    private float[] matrix;
+
+    /** Creates a new {@link Builder}. */
+    public Builder() {
+      matrix = GlUtil.create4x4IdentityMatrix();
+    }
+
+    /**
+     * Sets whether input overlay comes from an HDR source. If {@code true}, colors will be in
+     * linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
+     *
+     * <p>Set to {@code false} by default.
+     */
+    @CanIgnoreReturnValue
+    public Builder setUsesHdr(boolean useHdr) {
+      this.useHdr = useHdr;
+      return this;
+    }
+
+    /**
+     * Sets the {@link android.opengl.Matrix} used to transform the overlay before applying it to a
+     * frame.
+     *
+     * <p>Set to always return the identity matrix by default.
+     */
+    @CanIgnoreReturnValue
+    public Builder setMatrix(float[] matrix) {
+      this.matrix = matrix;
+      return this;
+    }
+
+    /** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
+    public OverlaySettings build() {
+      return new OverlaySettings(useHdr, matrix);
+    }
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+
+import android.content.Context;
+import android.opengl.GLES20;
+import android.opengl.Matrix;
+import android.util.Pair;
+import com.google.android.exoplayer2.util.FrameProcessingException;
+import com.google.android.exoplayer2.util.GlProgram;
+import com.google.android.exoplayer2.util.GlUtil;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+
+/** Applies one or more {@link TextureOverlay}s onto each frame. */
+/* package */ final class OverlayTextureProcessor extends SingleFrameGlTextureProcessor {
+
+  private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_overlay_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_overlay_es2.glsl";
+  private static final int MATRIX_OFFSET = 0;
+  private static final int TRANSPARENT_TEXTURE_WIDTH_HEIGHT = 1;
+
+  private final GlProgram glProgram;
+  private final ImmutableList<TextureOverlay> overlays;
+  private final float[] aspectRatioMatrix;
+  private final float[] overlayMatrix;
+
+  private int videoWidth;
+  private int videoHeight;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
+   *     in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
+   * @throws FrameProcessingException If a problem occurs while reading shader files.
+   */
+  public OverlayTextureProcessor(
+      Context context, boolean useHdr, ImmutableList<TextureOverlay> overlays)
+      throws FrameProcessingException {
+    super(useHdr);
+    checkArgument(!useHdr, "OverlayTextureProcessor does not support HDR colors yet.");
+    checkArgument(
+        overlays.size() <= 1,
+        "OverlayTextureProcessor does not support multiple overlays in the same processor yet.");
+    this.overlays = overlays;
+    aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
+    overlayMatrix = GlUtil.create4x4IdentityMatrix();
+
+    try {
+      glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
+    } catch (GlUtil.GlException | IOException e) {
+      throw new FrameProcessingException(e);
+    }
+
+    glProgram.setBufferAttribute(
+        "aFramePosition",
+        GlUtil.getNormalizedCoordinateBounds(),
+        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
+  }
+
+  @Override
+  public Pair<Integer, Integer> configure(int inputWidth, int inputHeight) {
+    videoWidth = inputWidth;
+    videoHeight = inputHeight;
+    return Pair.create(inputWidth, inputHeight);
+  }
+
+  @Override
+  public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
+    try {
+      glProgram.use();
+      if (!overlays.isEmpty()) {
+        TextureOverlay overlay = overlays.get(0);
+        glProgram.setSamplerTexIdUniform(
+            "uOverlayTexSampler1", overlay.getTextureId(presentationTimeUs), /* texUnitIndex= */ 1);
+        Pair<Integer, Integer> overlayTextureSize = overlay.getTextureSize(presentationTimeUs);
+        GlUtil.setToIdentity(aspectRatioMatrix);
+        Matrix.scaleM(
+            aspectRatioMatrix,
+            MATRIX_OFFSET,
+            videoWidth / (float) overlayTextureSize.first,
+            videoHeight / (float) overlayTextureSize.second,
+            /* z= */ 1);
+        glProgram.setFloatsUniform("uAspectRatioMatrix", aspectRatioMatrix);
+
+        Matrix.invertM(
+            overlayMatrix,
+            MATRIX_OFFSET,
+            overlay.getOverlaySettings(presentationTimeUs).matrix,
+            MATRIX_OFFSET);
+        glProgram.setFloatsUniform("uOverlayMatrix", overlayMatrix);
+
+      } else {
+        glProgram.setSamplerTexIdUniform(
+            "uOverlayTexSampler1", createTransparentTexture(), /* texUnitIndex= */ 1);
+      }
+      glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0);
+      glProgram.bindAttributesAndUniforms();
+      // The four-vertex triangle strip forms a quad.
+      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+      GlUtil.checkGlError();
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e, presentationTimeUs);
+    }
+  }
+
+  private int createTransparentTexture() throws FrameProcessingException {
+    try {
+      int textureId =
+          GlUtil.createTexture(
+              TRANSPARENT_TEXTURE_WIDTH_HEIGHT,
+              TRANSPARENT_TEXTURE_WIDTH_HEIGHT,
+              /* useHighPrecisionColorComponents= */ false);
+      GlUtil.checkGlError();
+      return textureId;
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+  }
+
+  @Override
+  public void release() throws FrameProcessingException {
+    super.release();
+    try {
+      glProgram.delete();
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/SimpleBitmapLoader.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package com.google.android.exoplayer2.effect;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.net.Uri;
+import androidx.annotation.Nullable;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.io.ByteStreams;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+// TODO(b/258685047): delete this copy once substitute is created in common
+//  (b/194284041, b/258658893)
+
+/**
+ * A {@link SimpleBitmapLoader} that delegates all tasks to an executor and supports fetching images
+ * from URIs with {@code file}, {@code http} and {@code https} schemes.
+ *
+ * <p>Loading tasks are delegated to an {@link ExecutorService} (or {@link
+ * ListeningExecutorService}) defined during construction. If no executor service is defined, all
+ * tasks are delegated to a single-thread executor service that is shared between instances of this
+ * class.
+ *
+ * <p>For HTTP(S) transfers, this class reads a resource only when the endpoint responds with an
+ * {@code HTTP 200} after sending the HTTP request.
+ */
+public final class SimpleBitmapLoader implements BitmapLoader {
+
+  private static final String FILE_URI_EXCEPTION_MESSAGE = "Could not read image from file";
+
+  private static final Supplier<ListeningExecutorService> DEFAULT_EXECUTOR_SERVICE =
+      Suppliers.memoize(
+          () -> MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()));
+
+  private final ListeningExecutorService executorService;
+
+  /**
+   * Creates an instance that delegates all load tasks to a single-thread executor service shared
+   * between instances.
+   */
+  public SimpleBitmapLoader() {
+    this(checkStateNotNull(DEFAULT_EXECUTOR_SERVICE.get()));
+  }
+
+  /** Creates an instance that delegates loading tasks to the {@code executorService}. */
+  public SimpleBitmapLoader(ExecutorService executorService) {
+    this.executorService = MoreExecutors.listeningDecorator(executorService);
+  }
+
+  @Override
+  public ListenableFuture<Bitmap> decodeBitmap(byte[] data) {
+    return executorService.submit(() -> decode(data));
+  }
+
+  @Override
+  public ListenableFuture<Bitmap> loadBitmap(Uri uri) {
+    return executorService.submit(() -> load(uri));
+  }
+
+  private static Bitmap decode(byte[] data) {
+    @Nullable Bitmap bitmap = BitmapFactory.decodeByteArray(data, /* offset= */ 0, data.length);
+    checkArgument(bitmap != null, "Could not decode image data");
+    return bitmap;
+  }
+
+  private static Bitmap load(Uri uri) throws IOException {
+    if ("file".equals(uri.getScheme())) {
+      @Nullable String path = uri.getPath();
+      if (path == null) {
+        throw new IllegalArgumentException(FILE_URI_EXCEPTION_MESSAGE);
+      }
+      @Nullable Bitmap bitmap = BitmapFactory.decodeFile(path);
+      if (bitmap == null) {
+        throw new IllegalArgumentException(FILE_URI_EXCEPTION_MESSAGE);
+      }
+      return bitmap;
+    }
+    URLConnection connection = new URL(uri.toString()).openConnection();
+    if (!(connection instanceof HttpURLConnection)) {
+      throw new UnsupportedOperationException("Unsupported scheme: " + uri.getScheme());
+    }
+    HttpURLConnection httpConnection = (HttpURLConnection) connection;
+    httpConnection.connect();
+    int responseCode = httpConnection.getResponseCode();
+    if (responseCode != HttpURLConnection.HTTP_OK) {
+      throw new IOException("Invalid response status code: " + responseCode);
+    }
+    try (InputStream inputStream = httpConnection.getInputStream()) {
+      return decode(ByteStreams.toByteArray(inputStream));
+    }
+  }
+}

library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureOverlay.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.effect;
+
+import android.util.Pair;
+import com.google.android.exoplayer2.util.FrameProcessingException;
+
+/** Creates overlays from OpenGL textures. */
+public abstract class TextureOverlay {
+  /**
+   * Returns the overlay texture identifier displayed at the specified timestamp.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   * @throws FrameProcessingException If an error occurs while processing or drawing the frame.
+   */
+  public abstract int getTextureId(long presentationTimeUs) throws FrameProcessingException;
+
+  // This method is required to find the size of a texture given a texture identifier using OpenGL
+  // ES 2.0. OpenGL ES 3.1 can do this with glGetTexLevelParameteriv().
+  /**
+   * Returns the pixel width and height of the overlay texture displayed at the specified timestamp.
+   *
+   * <p>This method must be called after {@link #getTextureId(long)}.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   */
+  public abstract Pair<Integer, Integer> getTextureSize(long presentationTimeUs);
+
+  /**
+   * Returns the {@link OverlaySettings} controlling how the overlay is displayed at the specified
+   * timestamp.
+   *
+   * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
+   */
+  public OverlaySettings getOverlaySettings(long presentationTimeUs) {
+    return new OverlaySettings.Builder().build();
+  }
+}