Abstract the interface of DefaultVideoFrameProcessor's input.

PiperOrigin-RevId: 526642898

Abstract the interface of DefaultVideoFrameProcessor's input.
PiperOrigin-RevId: 526642898
c284fac3 · claincly · Ian Baker · 11211620 · c284fac3 · c284fac3
Commit c284fac3 authored Apr 24, 2023 by claincly Committed by Ian Baker Apr 26, 2023
Showing with 136 additions and 64 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java
--- a/library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java
@@ -207,11 +207,8 @@ public interface VideoFrameProcessor {
  void registerInputFrame();
  /**
-   * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
+   * returns the number of input frames that have been made available to the {@code
-   * but not processed off the {@linkplain #getInputSurface() input surface} yet.
+   * VideoFrameProcessor} but have not been processed yet.
-   *
-   * <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
-   * Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
   *
   * <p>Can be called on any thread.
   */

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java
@@ -21,11 +21,13 @@ import static java.lang.Math.round;
 import android.graphics.Bitmap;
 import android.opengl.GLES20;
 import android.opengl.GLUtils;
+import android.view.Surface;
+import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.util.FrameInfo;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.VideoFrameProcessingException;
-import com.google.android.exoplayer2.util.VideoFrameProcessor;
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -36,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 *
 * <p>Public methods in this class can be called from any thread.
 */
-/* package */ final class BitmapTextureManager implements GlShaderProgram.InputListener {
+/* package */ final class BitmapTextureManager implements InputHandler {
  private final GlShaderProgram shaderProgram;
  private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
  // The queue holds all bitmaps with one or more frames pending to be sent downstream.
@@ -75,22 +77,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
        });
  }
-  /**
+  @Override
-   * Provides an input {@link Bitmap} to put into the video frames.
+  public void setDefaultBufferSize(int width, int height) {
-   *
+    throw new UnsupportedOperationException();
-   * @see VideoFrameProcessor#queueInputBitmap
+  }
-   */
+  @Override
  public void queueInputBitmap(
      Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
    videoFrameProcessingTaskExecutor.submit(
        () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
  }
-  /**
+  @Override
-   * Signals the end of the input.
+  public Surface getInputSurface() {
-   *
+    throw new UnsupportedOperationException();
-   * @see VideoFrameProcessor#signalEndOfInput()
+  }
-   */
+  @Override
+  public void registerInputFrame(FrameInfo frameInfo) {
+    // Do nothing.
+  }
+  @Override
+  public int getPendingFrameCount() {
+    // Always treat all queued bitmaps as immediately processed.
+    return 0;
+  }
+  @Override
  public void signalEndOfInput() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {
@@ -99,11 +114,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
        });
  }
-  /**
+  @Override
-   * Releases all resources.
+  public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
-   *
+    // Do nothing.
-   * @see VideoFrameProcessor#release()
+  }
-   */
+  @Override
  public void release() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
@@ -17,7 +17,6 @@ package com.google.android.exoplayer2.effect;
 import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
-import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 import static com.google.common.collect.Iterables.getLast;
@@ -245,8 +244,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
  private final EGLDisplay eglDisplay;
  private final EGLContext eglContext;
  private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
-  private @MonotonicNonNull BitmapTextureManager inputBitmapTextureManager;
+  private final InputHandler inputHandler;
-  private @MonotonicNonNull ExternalTextureManager inputExternalTextureManager;
  private final boolean releaseFramesAutomatically;
  private final FinalShaderProgramWrapper finalShaderProgramWrapper;
  private final ImmutableList<GlShaderProgram> allShaderPrograms;
@@ -276,20 +274,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
    switch (inputType) {
      case VideoFrameProcessor.INPUT_TYPE_SURFACE:
        checkState(inputShaderProgram instanceof ExternalShaderProgram);
-        inputExternalTextureManager =
+        inputHandler =
            new ExternalTextureManager(
                (ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
-        inputShaderProgram.setInputListener(inputExternalTextureManager);
        break;
      case VideoFrameProcessor.INPUT_TYPE_BITMAP:
-        inputBitmapTextureManager =
+        inputHandler =
            new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
-        inputShaderProgram.setInputListener(inputBitmapTextureManager);
        break;
      case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
      default:
        throw new VideoFrameProcessingException("Input type not supported yet");
    }
+    inputShaderProgram.setInputListener(inputHandler);
    finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
    allShaderPrograms = shaderPrograms;
@@ -317,18 +314,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
   * @param height The default height for input buffers, in pixels.
   */
  public void setInputDefaultBufferSize(int width, int height) {
-    checkNotNull(inputExternalTextureManager).setDefaultBufferSize(width, height);
+    inputHandler.setDefaultBufferSize(width, height);
  }
  @Override
  public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
-    checkNotNull(inputBitmapTextureManager)
+    inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
-        .queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
  }
  @Override
  public Surface getInputSurface() {
-    return checkNotNull(inputExternalTextureManager).getInputSurface();
+    return inputHandler.getInputSurface();
  }
  @Override
@@ -342,12 +338,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
    checkStateNotNull(
        nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
-    checkNotNull(inputExternalTextureManager).registerInputFrame(nextInputFrameInfo);
+    inputHandler.registerInputFrame(nextInputFrameInfo);
  }
  @Override
  public int getPendingInputFrameCount() {
-    return checkNotNull(inputExternalTextureManager).getPendingFrameCount();
+    return inputHandler.getPendingFrameCount();
  }
  @Override
@@ -368,12 +364,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
  public void signalEndOfInput() {
    checkState(!inputStreamEnded);
    inputStreamEnded = true;
-    if (inputBitmapTextureManager != null) {
+    videoFrameProcessingTaskExecutor.submit(inputHandler::signalEndOfInput);
-      videoFrameProcessingTaskExecutor.submit(inputBitmapTextureManager::signalEndOfInput);
-    }
-    if (inputExternalTextureManager != null) {
-      videoFrameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput);
-    }
  }
  @Override
@@ -381,10 +372,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
    try {
      videoFrameProcessingTaskExecutor.flush();
      CountDownLatch latch = new CountDownLatch(1);
-      checkNotNull(inputExternalTextureManager).setOnFlushCompleteListener(latch::countDown);
+      inputHandler.setOnFlushCompleteListener(latch::countDown);
      videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
      latch.await();
-      inputExternalTextureManager.setOnFlushCompleteListener(null);
+      inputHandler.setOnFlushCompleteListener(null);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
@@ -399,12 +390,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      Thread.currentThread().interrupt();
      throw new IllegalStateException(unexpected);
    }
-    if (inputExternalTextureManager != null) {
+    inputHandler.release();
-      inputExternalTextureManager.release();
-    }
-    if (inputBitmapTextureManager != null) {
-      inputBitmapTextureManager.release();
-    }
  }
  /**

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java
@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect;
 import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.annotation.Nullable;
@@ -26,7 +27,6 @@ import com.google.android.exoplayer2.util.FrameInfo;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.VideoFrameProcessingException;
-import com.google.android.exoplayer2.util.VideoFrameProcessor;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
 * {@link ExternalShaderProgram} for consumption.
 */
-/* package */ final class ExternalTextureManager implements InputListener {
+/* package */ final class ExternalTextureManager implements InputHandler {
  private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
  private final ExternalShaderProgram externalShaderProgram;
@@ -105,12 +105,18 @@ import java.util.concurrent.atomic.AtomicInteger;
    surface = new Surface(surfaceTexture);
  }
-  /** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */
+  @Override
  public void setDefaultBufferSize(int width, int height) {
    surfaceTexture.setDefaultBufferSize(width, height);
  }
-  /** Returns the {@linkplain Surface input surface} that wraps the external texture. */
+  @Override
+  public void queueInputBitmap(
+      Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
+    throw new UnsupportedOperationException();
+  }
+  @Override
  public Surface getInputSurface() {
    return surface;
  }
@@ -137,7 +143,7 @@ import java.util.concurrent.atomic.AtomicInteger;
        });
  }
-  /** Sets the task to run on completing flushing, or {@code null} to clear any task. */
+  @Override
  public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
    onFlushCompleteTask = task;
  }
@@ -154,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger;
   * <p>Can be called on any thread. The caller must ensure that frames are registered in the
   * correct order.
   */
+  @Override
  public void registerInputFrame(FrameInfo frame) {
    pendingFrames.add(frame);
  }
@@ -164,15 +171,12 @@ import java.util.concurrent.atomic.AtomicInteger;
   *
   * <p>Can be called on any thread.
   */
+  @Override
  public int getPendingFrameCount() {
    return pendingFrames.size();
  }
-  /**
+  @Override
-   * Signals the end of the input.
-   *
-   * @see VideoFrameProcessor#signalEndOfInput()
-   */
  public void signalEndOfInput() {
    videoFrameProcessingTaskExecutor.submit(
        () -> {
@@ -183,11 +187,7 @@ import java.util.concurrent.atomic.AtomicInteger;
        });
  }
-  /**
+  @Override
-   * Releases all resources.
-   *
-   * @see VideoFrameProcessor#release()
-   */
  public void release() {
    surfaceTexture.release();
    surface.release();

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package com.google.android.exoplayer2.effect;
+import android.graphics.Bitmap;
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.util.FrameInfo;
+import com.google.android.exoplayer2.util.VideoFrameProcessor;
+/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */
+/* package */ interface InputHandler extends GlShaderProgram.InputListener {
+  /**
+   * See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
+   *
+   * <p>Only works when the input is received on a {@link SurfaceTexture}.
+   */
+  void setDefaultBufferSize(int width, int height);
+  /**
+   * Provides an input {@link Bitmap} to put into the video frames.
+   *
+   * @see VideoFrameProcessor#queueInputBitmap
+   */
+  void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr);
+  /**
+   * See {@link VideoFrameProcessor#getInputSurface}.
+   *
+   * <p>Only works when the input is received on a {@link SurfaceTexture}.
+   */
+  Surface getInputSurface();
+  /** Informs the {@code InputHandler} that a frame will be queued. */
+  void registerInputFrame(FrameInfo frameInfo);
+  /** See {@link VideoFrameProcessor#getPendingInputFrameCount}. */
+  int getPendingFrameCount();
+  /**
+   * Signals the end of the input.
+   *
+   * @see VideoFrameProcessor#signalEndOfInput()
+   */
+  void signalEndOfInput();
+  /** Sets the task to run on completing flushing, or {@code null} to clear any task. */
+  void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task);
+  /**
+   * Releases all resources.
+   *
+   * @see VideoFrameProcessor#release()
+   */
+  void release();
+}