Allow setting individual offset for bitmaps.

PiperOrigin-RevId: 527001582

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java

@@ -66,11 +66,20 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
     videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
 
     videoFrameProcessorTestRunner.queueInputBitmap(
-        readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2);
+        readBitmap(ORIGINAL_PNG_ASSET_PATH),
+        C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ 0L,
+        /* frameRate= */ 2);
     videoFrameProcessorTestRunner.queueInputBitmap(
-        readBitmap(SCALE_WIDE_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3);
+        readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
+        2 * C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ 0L,
+        /* frameRate= */ 3);
     videoFrameProcessorTestRunner.queueInputBitmap(
-        readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4);
+        readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH),
+        3 * C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ 0L,
+        /* frameRate= */ 4);
     videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
 
     int actualFrameCount = framesProduced.get();
@@ -87,6 +96,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
       videoFrameProcessorTestRunner.queueInputBitmap(
           readBitmap(ORIGINAL_PNG_ASSET_PATH),
           /* durationUs= */ C.MICROS_PER_SECOND,
+          /* offsetToAddUs= */ 0L,
           /* frameRate= */ 1);
     }
     videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
@@ -97,6 +107,63 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
 
   @RequiresNonNull("framesProduced")
   @Test
+  public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs()
+      throws Exception {
+    String testId =
+        "imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs";
+    Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
+    videoFrameProcessorTestRunner =
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
+            .build();
+
+    long offsetUs = 1_000_000L;
+    videoFrameProcessorTestRunner.queueInputBitmap(
+        readBitmap(ORIGINAL_PNG_ASSET_PATH),
+        /* durationUs= */ C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ offsetUs,
+        /* frameRate= */ 2);
+    videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
+    assertThat(actualPresentationTimesUs)
+        .containsExactly(offsetUs, offsetUs + C.MICROS_PER_SECOND / 2)
+        .inOrder();
+  }
+
+  @RequiresNonNull("framesProduced")
+  @Test
+  public void imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs()
+      throws Exception {
+    String testId = "imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs";
+    Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
+    videoFrameProcessorTestRunner =
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
+            .build();
+
+    long offsetUs1 = 1_000_000L;
+    videoFrameProcessorTestRunner.queueInputBitmap(
+        readBitmap(ORIGINAL_PNG_ASSET_PATH),
+        /* durationUs= */ C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ offsetUs1,
+        /* frameRate= */ 2);
+    long offsetUs2 = 2_000_000L;
+    videoFrameProcessorTestRunner.queueInputBitmap(
+        readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
+        /* durationUs= */ C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ offsetUs2,
+        /* frameRate= */ 2);
+    videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
+    assertThat(actualPresentationTimesUs)
+        .containsExactly(
+            offsetUs1,
+            offsetUs1 + C.MICROS_PER_SECOND / 2,
+            offsetUs2,
+            offsetUs2 + C.MICROS_PER_SECOND / 2)
+        .inOrder();
+  }
+
+  @RequiresNonNull("framesProduced")
+  @Test
   public void
       imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs()
           throws Exception {
@@ -111,11 +178,13 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
     videoFrameProcessorTestRunner.queueInputBitmap(
         readBitmap(ORIGINAL_PNG_ASSET_PATH),
         /* durationUs= */ C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ 0L,
         /* frameRate= */ 2);
     videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
     videoFrameProcessorTestRunner.queueInputBitmap(
         readBitmap(ORIGINAL_PNG_ASSET_PATH),
         /* durationUs= */ 2 * C.MICROS_PER_SECOND,
+        /* offsetToAddUs= */ 0L,
         /* frameRate= */ 3);
 
     assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder();

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java

@@ -120,7 +120,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
     Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH);
 
     videoFrameProcessorTestRunner.queueInputBitmap(
-        originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
+        originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
     Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
 
     // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
@@ -148,7 +148,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
     Bitmap expectedBitmap = readBitmap(IMAGE_TO_CROPPED_VIDEO_PNG_ASSET_PATH);
 
     videoFrameProcessorTestRunner.queueInputBitmap(
-        originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
+        originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
     Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
 
     // TODO(b/207848601): Switch to using proper tooling for testing against golden data.

library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java

@@ -77,9 +77,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   @Override
   public void queueInputBitmap(
-      Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
+      Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
     videoFrameProcessingTaskExecutor.submit(
-        () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
+        () -> setupBitmap(inputBitmap, durationUs, offsetUs, frameRate, useHdr));
   }
 
   @Override
@@ -114,7 +114,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   // Methods that must be called on the GL thread.
 
-  private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr)
+  private void setupBitmap(
+      Bitmap bitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr)
       throws VideoFrameProcessingException {
     this.useHdr = useHdr;
     if (inputEnded) {
@@ -122,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
     int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
     double frameDurationUs = C.MICROS_PER_SECOND / frameRate;
-    pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, frameDurationUs, framesToAdd));
+    pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, offsetUs, frameDurationUs, framesToAdd));
 
     maybeQueueToShaderProgram();
   }
@@ -136,6 +137,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     if (framesToQueueForCurrentBitmap == 0) {
       Bitmap bitmap = currentBitmapInfo.bitmap;
       framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames;
+      currentPresentationTimeUs = currentBitmapInfo.offsetUs;
       int currentTexId;
       try {
         if (currentGlTextureInfo != null) {
@@ -187,11 +189,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   /** Information to generate all the frames associated with a specific {@link Bitmap}. */
   private static final class BitmapFrameSequenceInfo {
     public final Bitmap bitmap;
+    public final long offsetUs;
     public final double frameDurationUs;
     public final int numberOfFrames;
 
-    public BitmapFrameSequenceInfo(Bitmap bitmap, double frameDurationUs, int numberOfFrames) {
+    public BitmapFrameSequenceInfo(
+        Bitmap bitmap, long offsetUs, double frameDurationUs, int numberOfFrames) {
       this.bitmap = bitmap;
+      this.offsetUs = offsetUs;
       this.frameDurationUs = frameDurationUs;
       this.numberOfFrames = numberOfFrames;
     }

library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java

@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect;
 
 import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 import static com.google.common.collect.Iterables.getLast;
@@ -251,6 +252,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
 
   private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
   private volatile boolean inputStreamEnded;
+  private volatile boolean hasRefreshedNextInputFrameInfo;
 
   private DefaultVideoFrameProcessor(
       EGLDisplay eglDisplay,
@@ -319,7 +321,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
 
   @Override
   public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
-    inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
+    checkState(
+        hasRefreshedNextInputFrameInfo,
+        "setInputFrameInfo must be called before queueing another bitmap");
+    inputHandler.queueInputBitmap(
+        inputBitmap,
+        durationUs,
+        checkNotNull(nextInputFrameInfo).offsetToAddUs,
+        frameRate,
+        /* useHdr= */ false);
+    hasRefreshedNextInputFrameInfo = false;
   }
 
   @Override
@@ -330,6 +341,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
   @Override
   public void setInputFrameInfo(FrameInfo inputFrameInfo) {
     nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
+    hasRefreshedNextInputFrameInfo = true;
   }
 
   @Override
@@ -339,6 +351,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
         nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
 
     inputHandler.registerInputFrame(nextInputFrameInfo);
+    hasRefreshedNextInputFrameInfo = false;
   }
 
   @Override

library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java

@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect;
 
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.annotation.Nullable;
@@ -110,6 +111,12 @@ import java.util.concurrent.atomic.AtomicInteger;
   }
 
   @Override
+  public void queueInputBitmap(
+      Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
   public Surface getInputSurface() {
     return surface;
   }

library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java

@@ -39,10 +39,16 @@ import com.google.android.exoplayer2.util.VideoFrameProcessor;
   /**
    * Provides an input {@link Bitmap} to put into the video frames.
    *
-   * @see VideoFrameProcessor#queueInputBitmap
+   * @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}.
+   * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
+   * @param offsetUs The offset, from the start of the input stream, to apply for the {@code
+   *     inputBitmap} in microseconds.
+   * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
+   *     second.
+   * @param useHdr Whether input and/or output colors are HDR.
    */
   default void queueInputBitmap(
-      Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
+      Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
     throw new UnsupportedOperationException();
   }
 

testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java

@@ -331,10 +331,12 @@ public final class VideoFrameProcessorTestRunner {
     return endFrameProcessingAndGetImage();
   }
 
-  public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
+  public void queueInputBitmap(
+      Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) {
     videoFrameProcessor.setInputFrameInfo(
         new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
             .setPixelWidthHeightRatio(pixelWidthHeightRatio)
+            .setOffsetToAddUs(offsetToAddUs)
             .build());
     videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
   }