Avoid spinning while queueing input to ExternalTextureProcessor.

This change adds ExternalTextureManager which implements
InputListener to only queue input frames to the
ExternalTextureProcessor when it is ready to accept an input
frame. This replaces the old retry-logic in GlEffectsFrameProcessor.

Before this change, the retrying in GlEffectFrameProcessor wasted
CPU time if input becomes available faster than the
ExternalTextureProcessor can process it.

PiperOrigin-RevId: 467177659

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import android.graphics.SurfaceTexture;
+import androidx.annotation.Nullable;
+import androidx.annotation.WorkerThread;
+import androidx.media3.common.FrameInfo;
+import androidx.media3.common.FrameProcessingException;
+import androidx.media3.common.FrameProcessor;
+import androidx.media3.effect.GlTextureProcessor.InputListener;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.util.GlUtil;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
+ * {@link ExternalTextureProcessor} for consumption.
+ */
+/* package */ class ExternalTextureManager implements InputListener {
+
+  private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
+  private final ExternalTextureProcessor externalTextureProcessor;
+  private final int externalTexId;
+  private final SurfaceTexture surfaceTexture;
+  private final float[] textureTransformMatrix;
+  private final Queue<FrameInfo> pendingFrames;
+
+  // Incremented on any thread, decremented on the GL thread only.
+  private final AtomicInteger availableFrameCount;
+  // Incremented on any thread, decremented on the GL thread only.
+  private final AtomicInteger externalTextureProcessorInputCapacity;
+
+  // Set to true on any thread. Read on the GL thread only.
+  private volatile boolean inputStreamEnded;
+  // Set to null on any thread. Read and set to non-null on the GL thread only.
+  @Nullable private volatile FrameInfo frame;
+
+  private long previousStreamOffsetUs;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param externalTextureProcessor The {@link ExternalTextureProcessor} for which this {@code
+   *     ExternalTextureManager} will be set as the {@link InputListener}.
+   * @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor}.
+   * @throws FrameProcessingException If a problem occurs while creating the external texture.
+   */
+  public ExternalTextureManager(
+      ExternalTextureProcessor externalTextureProcessor,
+      FrameProcessingTaskExecutor frameProcessingTaskExecutor)
+      throws FrameProcessingException {
+    this.externalTextureProcessor = externalTextureProcessor;
+    this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
+    try {
+      externalTexId = GlUtil.createExternalTexture();
+    } catch (GlUtil.GlException e) {
+      throw new FrameProcessingException(e);
+    }
+    surfaceTexture = new SurfaceTexture(externalTexId);
+    textureTransformMatrix = new float[16];
+    pendingFrames = new ConcurrentLinkedQueue<>();
+    availableFrameCount = new AtomicInteger();
+    externalTextureProcessorInputCapacity = new AtomicInteger();
+    previousStreamOffsetUs = C.TIME_UNSET;
+  }
+
+  public SurfaceTexture getSurfaceTexture() {
+    surfaceTexture.setOnFrameAvailableListener(
+        unused -> {
+          availableFrameCount.getAndIncrement();
+          frameProcessingTaskExecutor.submit(
+              () -> {
+                if (maybeUpdateFrame()) {
+                  maybeQueueFrameToExternalTextureProcessor();
+                }
+              });
+        });
+    return surfaceTexture;
+  }
+
+  @Override
+  public void onReadyToAcceptInputFrame() {
+    externalTextureProcessorInputCapacity.getAndIncrement();
+    frameProcessingTaskExecutor.submit(this::maybeQueueFrameToExternalTextureProcessor);
+  }
+
+  @Override
+  public void onInputFrameProcessed(TextureInfo inputTexture) {
+    frame = null;
+    frameProcessingTaskExecutor.submit(
+        () -> {
+          if (maybeUpdateFrame()) {
+            maybeQueueFrameToExternalTextureProcessor();
+          }
+        });
+  }
+
+  /**
+   * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
+   * become available via the {@link SurfaceTexture} eventually.
+   *
+   * <p>Can be called on any thread, but the caller must ensure that frames are registered in the
+   * correct order.
+   */
+  public void registerInputFrame(FrameInfo frame) {
+    pendingFrames.add(frame);
+  }
+
+  /**
+   * Returns the number of {@linkplain #registerInputFrame(FrameInfo) registered} frames that have
+   * not been rendered to the external texture yet.
+   *
+   * <p>Can be called on any thread.
+   */
+  public int getPendingFrameCount() {
+    return pendingFrames.size();
+  }
+
+  /**
+   * Signals the end of the input.
+   *
+   * @see FrameProcessor#signalEndOfInput()
+   */
+  @WorkerThread
+  public void signalEndOfInput() {
+    inputStreamEnded = true;
+    if (pendingFrames.isEmpty() && frame == null) {
+      externalTextureProcessor.signalEndOfCurrentInputStream();
+    }
+  }
+
+  public void release() {
+    surfaceTexture.release();
+  }
+
+  @WorkerThread
+  private boolean maybeUpdateFrame() {
+    if (frame != null || availableFrameCount.get() == 0) {
+      return false;
+    }
+
+    availableFrameCount.getAndDecrement();
+    surfaceTexture.updateTexImage();
+    frame = pendingFrames.remove();
+    return true;
+  }
+
+  @WorkerThread
+  private void maybeQueueFrameToExternalTextureProcessor() {
+    if (externalTextureProcessorInputCapacity.get() == 0 || frame == null) {
+      return;
+    }
+
+    FrameInfo frame = this.frame;
+    externalTextureProcessorInputCapacity.getAndDecrement();
+    surfaceTexture.getTransformMatrix(textureTransformMatrix);
+    externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
+    long frameTimeNs = surfaceTexture.getTimestamp();
+    long streamOffsetUs = frame.streamOffsetUs;
+    if (streamOffsetUs != previousStreamOffsetUs) {
+      if (previousStreamOffsetUs != C.TIME_UNSET) {
+        externalTextureProcessor.signalEndOfCurrentInputStream();
+      }
+      previousStreamOffsetUs = streamOffsetUs;
+    }
+    // Correct for the stream offset so processors see original media presentation timestamps.
+    long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
+    externalTextureProcessor.queueInputFrame(
+        new TextureInfo(externalTexId, /* fboId= */ C.INDEX_UNSET, frame.width, frame.height),
+        presentationTimeUs);
+
+    if (inputStreamEnded && pendingFrames.isEmpty()) {
+      externalTextureProcessor.signalEndOfCurrentInputStream();
+    }
+  }
+}

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureProcessor.java

@@ -31,11 +31,4 @@ package androidx.media3.effect;
    *     android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
    *     android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
    */
    */
   void setTextureTransformMatrix(float[] textureTransformMatrix);
   void setTextureTransformMatrix(float[] textureTransformMatrix);
-
-  /**
-   * Returns whether another input frame can be {@linkplain #queueInputFrame(TextureInfo, long)
-   * queued}.
-   */
-  // TODO(b/227625423): Remove this method and use the input listener instead.
-  boolean acceptsInputFrame();
 }
 }

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java

@@ -42,8 +42,8 @@ import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList;
-import java.util.ArrayDeque;
 import java.util.Queue;
 import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 
@@ -112,7 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 
     textureTransformMatrix = new float[16];
     textureTransformMatrix = new float[16];
     Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
     Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
-    streamOffsetUsQueue = new ArrayDeque<>();
+    streamOffsetUsQueue = new ConcurrentLinkedQueue<>();
     inputListener = new InputListener() {};
     inputListener = new InputListener() {};
   }
   }
 
 
@@ -135,11 +135,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
   }
 
 
   @Override
   @Override
-  public boolean acceptsInputFrame() {
-    return true;
-  }
-
-  @Override
   public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
   public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
     checkState(!streamOffsetUsQueue.isEmpty(), "No input stream specified.");
     checkState(!streamOffsetUsQueue.isEmpty(), "No input stream specified.");
 
 
@@ -329,12 +324,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    * Signals that there will be another input stream after all previously appended input streams
    * Signals that there will be another input stream after all previously appended input streams
    * have {@linkplain #signalEndOfCurrentInputStream() ended}.
    * have {@linkplain #signalEndOfCurrentInputStream() ended}.
    *
    *
+   * <p>This method does not need to be called on the GL thread, but the caller must ensure that
+   * stream offsets are appended in the correct order.
+   *
    * @param streamOffsetUs The presentation timestamp offset, in microseconds.
    * @param streamOffsetUs The presentation timestamp offset, in microseconds.
    */
    */
   public void appendStream(long streamOffsetUs) {
   public void appendStream(long streamOffsetUs) {
     streamOffsetUsQueue.add(streamOffsetUs);
     streamOffsetUsQueue.add(streamOffsetUs);
   }
   }
 
 
+  /**
+   * Sets the output {@link SurfaceInfo}.
+   *
+   * @see FrameProcessor#setOutputSurfaceInfo(SurfaceInfo)
+   */
   public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
   public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
     if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
     if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
       if (outputSurfaceInfo != null
       if (outputSurfaceInfo != null

google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/SingleFrameGlTextureProcessor.java

@@ -106,10 +106,6 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
     this.errorListener = errorListener;
     this.errorListener = errorListener;
   }
   }
 
 
-  public final boolean acceptsInputFrame() {
-    return !outputTextureInUse;
-  }
-
   @Override
   @Override
   public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
   public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
     checkState(
     checkState(