Add support for customizing overlay anchor point.

PiperOrigin-RevId: 496956035

Add support for customizing overlay anchor point.
PiperOrigin-RevId: 496956035
4d1283f4 · tofunmi · Marc Baechinger · 890fd0a9 · 4d1283f4 · 4d1283f4
Commit 4d1283f4 authored Dec 21, 2022 by tofunmi Committed by Marc Baechinger Jan 04, 2023
Showing with 98 additions and 12 deletions
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java
testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png
--- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java
+++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java
@@ -62,6 +62,8 @@ public class OverlayTextureProcessorPixelTest {
      "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
  public static final String OVERLAY_BITMAP_DEFAULT =
      "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png";
+  public static final String OVERLAY_BITMAP_ANCHORED =
+      "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png";
  public static final String OVERLAY_BITMAP_SCALED =
      "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png";
  public static final String OVERLAY_BITMAP_TRANSLUCENT =
@@ -174,6 +176,33 @@ public class OverlayTextureProcessorPixelTest {
  }
  @Test
+  public void drawFrame_anchoredBitmapOverlay_blendsBitmapIntoTopLeftOfFrame() throws Exception {
+    String testId = "drawFrame_anchoredBitmapOverlay";
+    Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
+    float[] translateMatrix = GlUtil.create4x4IdentityMatrix();
+    Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ -1f, /* y= */ 1f, /* z= */ 1);
+    OverlaySettings overlaySettings =
+        new OverlaySettings.Builder().setMatrix(translateMatrix).setAnchor(-1f, 1f).build();
+    BitmapOverlay staticBitmapOverlay =
+        BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
+    overlayTextureProcessor =
+        new OverlayEffect(ImmutableList.of(staticBitmapOverlay))
+            .toGlTextureProcessor(context, /* useHdr= */ false);
+    Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
+    setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
+    Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ANCHORED);
+    overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
+    Bitmap actualBitmap =
+        createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
+    maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
+    float averagePixelAbsoluteDifference =
+        getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+  @Test
  public void drawFrame_translucentBitmapOverlay_blendsBitmapIntoFrame() throws Exception {
    String testId = "drawFrame_translucentBitmapOverlay";
    Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java
@@ -16,6 +16,7 @@ package com.google.android.exoplayer2.effect;
 */
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+import android.util.Pair;
 import androidx.annotation.FloatRange;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
@@ -25,11 +26,13 @@ public final class OverlaySettings {
  public final boolean useHdr;
  public final float alpha;
  public final float[] matrix;
+  public final Pair<Float, Float> anchor;
-  private OverlaySettings(boolean useHdr, float alpha, float[] matrix) {
+  private OverlaySettings(boolean useHdr, float alpha, float[] matrix, Pair<Float, Float> anchor) {
    this.useHdr = useHdr;
    this.alpha = alpha;
    this.matrix = matrix;
+    this.anchor = anchor;
  }
  /** A builder for {@link OverlaySettings} instances. */
@@ -37,10 +40,12 @@ public final class OverlaySettings {
    private boolean useHdr;
    private float alpha = 1;
    private float[] matrix;
+    private Pair<Float, Float> anchor;
    /** Creates a new {@link Builder}. */
    public Builder() {
      matrix = GlUtil.create4x4IdentityMatrix();
+      anchor = Pair.create(0f, 0f);
    }
    /**
@@ -81,9 +86,33 @@ public final class OverlaySettings {
      return this;
    }
+    /**
+     * Sets the coordinates for the anchor point of the overlay.
+     *
+     * <p>The anchor point is the point inside the overlay that the overlay is positioned from.
+     *
+     * <p>The coordinates are specified in Normalised Device Coordinates (NDCs). Set to always
+     * return {@code (0,0)} (the center) by default.
+     *
+     * @param x the NDC x-coordinate.
+     * @param y the NDC y-coordinate.
+     */
+    @CanIgnoreReturnValue
+    public Builder setAnchor(
+        @FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
+      checkArgument(
+          -1 <= x && x <= 1,
+          "x needs to be specified in terms of NDCs which lie in the interval [-1, 1].");
+      checkArgument(
+          -1 <= y && y <= 1,
+          "y needs to be specified in terms of NDCs which lie in the interval [-1, 1].");
+      this.anchor = Pair.create(x, y);
+      return this;
+    }
    /** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
    public OverlaySettings build() {
-      return new OverlaySettings(useHdr, alpha, matrix);
+      return new OverlaySettings(useHdr, alpha, matrix, anchor);
    }
  }
 }
--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java
@@ -20,6 +20,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import android.content.Context;
 import android.opengl.GLES20;
 import android.opengl.Matrix;
+import android.util.Pair;
 import com.google.android.exoplayer2.util.FrameProcessingException;
 import com.google.android.exoplayer2.util.GlProgram;
 import com.google.android.exoplayer2.util.GlUtil;
@@ -36,6 +37,8 @@ import com.google.common.collect.ImmutableList;
  private final ImmutableList<TextureOverlay> overlays;
  private final float[] aspectRatioMatrix;
  private final float[] overlayMatrix;
+  private final float[] anchorMatrix;
+  private final float[] transformationMatrix;
  private int videoWidth;
  private int videoHeight;
@@ -61,7 +64,8 @@ import com.google.common.collect.ImmutableList;
    this.overlays = overlays;
    aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
    overlayMatrix = GlUtil.create4x4IdentityMatrix();
+    anchorMatrix = GlUtil.create4x4IdentityMatrix();
+    transformationMatrix = GlUtil.create4x4IdentityMatrix();
    try {
      glProgram =
          new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size()));
@@ -89,10 +93,12 @@ import com.google.common.collect.ImmutableList;
      if (!overlays.isEmpty()) {
        for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
          TextureOverlay overlay = overlays.get(texUnitIndex - 1);
          glProgram.setSamplerTexIdUniform(
              Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex),
              overlay.getTextureId(presentationTimeUs),
              texUnitIndex);
          GlUtil.setToIdentity(aspectRatioMatrix);
          Matrix.scaleM(
              aspectRatioMatrix,
@@ -100,15 +106,40 @@ import com.google.common.collect.ImmutableList;
              videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(),
              videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(),
              /* z= */ 1);
-          glProgram.setFloatsUniform(
-              Util.formatInvariant("uAspectRatioMatrix%d", texUnitIndex), aspectRatioMatrix);
          Matrix.invertM(
              overlayMatrix,
              MATRIX_OFFSET,
              overlay.getOverlaySettings(presentationTimeUs).matrix,
              MATRIX_OFFSET);
+          Pair<Float, Float> overlayAnchor = overlay.getOverlaySettings(presentationTimeUs).anchor;
+          GlUtil.setToIdentity(anchorMatrix);
+          Matrix.translateM(
+              anchorMatrix,
+              /* mOffset= */ 0,
+              overlayAnchor.first
+                  * overlay.getTextureSize(presentationTimeUs).getWidth()
+                  / videoWidth,
+              overlayAnchor.second
+                  * overlay.getTextureSize(presentationTimeUs).getHeight()
+                  / videoHeight,
+              /* z= */ 1);
+          Matrix.multiplyMM(
+              transformationMatrix,
+              MATRIX_OFFSET,
+              overlayMatrix,
+              MATRIX_OFFSET,
+              anchorMatrix,
+              MATRIX_OFFSET);
+          Matrix.multiplyMM(
+              transformationMatrix,
+              MATRIX_OFFSET,
+              aspectRatioMatrix,
+              MATRIX_OFFSET,
+              transformationMatrix,
+              MATRIX_OFFSET);
          glProgram.setFloatsUniform(
-              Util.formatInvariant("uOverlayMatrix%d", texUnitIndex), overlayMatrix);
+              Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), transformationMatrix);
          glProgram.setFloatUniform(
              Util.formatInvariant("uOverlayAlpha%d", texUnitIndex),
              overlay.getOverlaySettings(presentationTimeUs).alpha);
@@ -143,9 +174,8 @@ import com.google.common.collect.ImmutableList;
    for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
      shader
-          .append(Util.formatInvariant("uniform mat4 uAspectRatioMatrix%d;\n", texUnitIndex))
+          .append(Util.formatInvariant("uniform mat4 uTransformationMatrix%s;\n", texUnitIndex))
-          .append(Util.formatInvariant("uniform mat4 uOverlayMatrix%d;\n", texUnitIndex))
+          .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%s;\n", texUnitIndex));
-          .append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex));
    }
    shader
@@ -160,9 +190,7 @@ import com.google.common.collect.ImmutableList;
      shader
          .append(Util.formatInvariant("  vec4 aOverlayPosition%d = \n", texUnitIndex))
          .append(
-              Util.formatInvariant(
+              Util.formatInvariant("  uTransformationMatrix%s * aFramePosition;\n", texUnitIndex))
-                  "    uAspectRatioMatrix%d * uOverlayMatrix%d * aFramePosition;\n",
-                  texUnitIndex, texUnitIndex))
          .append(
              Util.formatInvariant(
                  "  vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n",

--- a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png
+++ b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png