Allow FrameProcessor input surface size changes.

This will be useful for downgrading to a lower resolution during
a slow preview and for processing slide-shows once sequential
multi-asset editing is supported.

PiperOrigin-RevId: 457017255
(cherry picked from commit a230d59f)

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java

@@ -367,8 +367,6 @@ public final class GlEffectsFrameProcessorPixelTest {
                     }
                   },
                   pixelWidthHeightRatio,
-                  inputWidth,
-                  inputHeight,
                   /* streamOffsetUs= */ 0L,
                   effects,
                   /* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
@@ -383,6 +381,7 @@ public final class GlEffectsFrameProcessorPixelTest {
                   },
                   Transformer.DebugViewProvider.NONE,
                   /* enableExperimentalHdrEditing= */ false));
+      glEffectsFrameProcessor.setInputFrameInfo(new FrameInfo(inputWidth, inputHeight));
       glEffectsFrameProcessor.registerInputFrame();
 
       // Queue the first video frame from the extractor.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameInfo.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
+
+/** Value class specifying information about a decoded video frame. */
+/* package */ class FrameInfo {
+  /** The width of the frame, in pixels. */
+  public final int width;
+  /** The height of the frame, in pixels. */
+  public final int height;
+
+  // TODO(b/227625423): Add pixelWidthHeightRatio.
+  // TODO(b/227624622): Add color space information for HDR.
+
+  public FrameInfo(int width, int height) {
+    checkArgument(width > 0, "width must be positive, but is: " + width);
+    checkArgument(height > 0, "height must be positive, but is: " + height);
+
+    this.width = width;
+    this.height = height;
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java

@@ -42,11 +42,20 @@ import android.view.Surface;
   Surface getInputSurface();
 
   /**
+   * Sets information about the input frames.
+   *
+   * <p>The new input information is applied from the next frame {@linkplain #registerInputFrame()
+   * registered} onwards.
+   */
+  void setInputFrameInfo(FrameInfo inputFrameInfo);
+
+  /**
    * Informs the {@code FrameProcessor} that a frame will be queued to its input surface.
    *
    * <p>Must be called before rendering a frame to the frame processor's input surface.
    *
-   * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
+   * @throws IllegalStateException If called after {@link #signalEndOfInputStream()} or before
+   *     {@link #setInputFrameInfo(FrameInfo)}.
    */
   void registerInputFrame();
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java

@@ -15,8 +15,8 @@
  */
 package com.google.android.exoplayer2.transformer;
 
-import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
 
 import android.content.Context;
 import android.graphics.SurfaceTexture;
@@ -31,10 +31,11 @@ import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Util;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
  * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
@@ -51,8 +52,6 @@ import java.util.concurrent.atomic.AtomicInteger;
    * @param listener A {@link Listener}.
    * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
    *     by this ratio so that the output frame's pixels have a ratio of 1.
-   * @param inputWidth The input frame width, in pixels.
-   * @param inputHeight The input frame height, in pixels.
    * @param effects The {@link GlEffect GlEffects} to apply to each frame.
    * @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link
    *     Surface}.
@@ -66,16 +65,12 @@ import java.util.concurrent.atomic.AtomicInteger;
       Context context,
       FrameProcessor.Listener listener,
       float pixelWidthHeightRatio,
-      int inputWidth,
-      int inputHeight,
       long streamOffsetUs,
       List<GlEffect> effects,
       SurfaceInfo.Provider outputSurfaceProvider,
       Transformer.DebugViewProvider debugViewProvider,
       boolean enableExperimentalHdrEditing)
       throws FrameProcessingException {
-    checkArgument(inputWidth > 0, "inputWidth must be positive");
-    checkArgument(inputHeight > 0, "inputHeight must be positive");
 
     ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
 
@@ -86,8 +81,6 @@ import java.util.concurrent.atomic.AtomicInteger;
                     context,
                     listener,
                     pixelWidthHeightRatio,
-                    inputWidth,
-                    inputHeight,
                     streamOffsetUs,
                     effects,
                     outputSurfaceProvider,
@@ -118,8 +111,6 @@ import java.util.concurrent.atomic.AtomicInteger;
       Context context,
       FrameProcessor.Listener listener,
       float pixelWidthHeightRatio,
-      int inputWidth,
-      int inputHeight,
       long streamOffsetUs,
       List<GlEffect> effects,
       SurfaceInfo.Provider outputSurfaceProvider,
@@ -177,8 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger;
         eglContext,
         frameProcessingTaskExecutor,
         streamOffsetUs,
-        /* inputExternalTexture= */ new TextureInfo(
-            GlUtil.createExternalTexture(), /* fboId= */ C.INDEX_UNSET, inputWidth, inputHeight),
+        /* inputExternalTextureId= */ GlUtil.createExternalTexture(),
         externalTextureProcessor,
         textureProcessors);
   }
@@ -300,21 +290,19 @@ import java.util.concurrent.atomic.AtomicInteger;
    * timestamps, in microseconds.
    */
   private final long streamOffsetUs;
-  /**
-   * Number of frames {@linkplain #registerInputFrame() registered} but not processed off the {@link
-   * #inputSurfaceTexture} yet.
-   */
-  private final AtomicInteger pendingInputFrameCount;
+
   /** Associated with an OpenGL external texture. */
   private final SurfaceTexture inputSurfaceTexture;
   /** Wraps the {@link #inputSurfaceTexture}. */
   private final Surface inputSurface;
 
   private final float[] inputSurfaceTextureTransformMatrix;
-  private final TextureInfo inputExternalTexture;
+  private final int inputExternalTextureId;
   private final ExternalTextureProcessor inputExternalTextureProcessor;
   private final ImmutableList<GlTextureProcessor> textureProcessors;
+  private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
 
+  private @MonotonicNonNull FrameInfo nextInputFrameInfo;
   private boolean inputStreamEnded;
 
   private GlEffectsFrameProcessor(
@@ -322,7 +310,7 @@ import java.util.concurrent.atomic.AtomicInteger;
       EGLContext eglContext,
       FrameProcessingTaskExecutor frameProcessingTaskExecutor,
       long streamOffsetUs,
-      TextureInfo inputExternalTexture,
+      int inputExternalTextureId,
       ExternalTextureProcessor inputExternalTextureProcessor,
       ImmutableList<GlTextureProcessor> textureProcessors) {
     checkState(!textureProcessors.isEmpty());
@@ -331,33 +319,40 @@ import java.util.concurrent.atomic.AtomicInteger;
     this.eglContext = eglContext;
     this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
     this.streamOffsetUs = streamOffsetUs;
-    this.inputExternalTexture = inputExternalTexture;
+    this.inputExternalTextureId = inputExternalTextureId;
     this.inputExternalTextureProcessor = inputExternalTextureProcessor;
     this.textureProcessors = textureProcessors;
 
-    pendingInputFrameCount = new AtomicInteger();
-    inputSurfaceTexture = new SurfaceTexture(inputExternalTexture.texId);
+    inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
     inputSurface = new Surface(inputSurfaceTexture);
     inputSurfaceTextureTransformMatrix = new float[16];
+    pendingInputFrames = new ConcurrentLinkedQueue<>();
   }
 
   @Override
   public Surface getInputSurface() {
-    // TODO(b/227625423): Allow input surface to be recreated for input size change.
     inputSurfaceTexture.setOnFrameAvailableListener(
         surfaceTexture -> frameProcessingTaskExecutor.submit(this::processInputFrame));
     return inputSurface;
   }
 
   @Override
+  public void setInputFrameInfo(FrameInfo inputFrameInfo) {
+    nextInputFrameInfo = inputFrameInfo;
+  }
+
+  @Override
   public void registerInputFrame() {
     checkState(!inputStreamEnded);
-    pendingInputFrameCount.incrementAndGet();
+    checkStateNotNull(
+        nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
+
+    pendingInputFrames.add(nextInputFrameInfo);
   }
 
   @Override
   public int getPendingInputFrameCount() {
-    return pendingInputFrameCount.get();
+    return pendingInputFrames.size();
   }
 
   @Override
@@ -401,10 +396,15 @@ import java.util.concurrent.atomic.AtomicInteger;
     long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
     inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
     inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
+    FrameInfo inputFrameInfo = pendingInputFrames.remove();
     checkState(
         inputExternalTextureProcessor.maybeQueueInputFrame(
-            inputExternalTexture, presentationTimeUs));
-    checkState(pendingInputFrameCount.getAndDecrement() > 0);
+            new TextureInfo(
+                inputExternalTextureId,
+                /* fboId= */ C.INDEX_UNSET,
+                inputFrameInfo.width,
+                inputFrameInfo.height),
+            presentationTimeUs));
     // After the inputExternalTextureProcessor has produced an output frame, it is processed
     // asynchronously by the texture processors chained after it.
   }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -120,8 +120,6 @@ import org.checkerframework.dataflow.qual.Pure;
                 }
               },
               inputFormat.pixelWidthHeightRatio,
-              /* inputWidth= */ decodedWidth,
-              /* inputHeight= */ decodedHeight,
               streamOffsetUs,
               effectsListBuilder.build(),
               /* outputSurfaceProvider= */ encoderWrapper,
@@ -131,6 +129,7 @@ import org.checkerframework.dataflow.qual.Pure;
       throw TransformationException.createForFrameProcessingException(
           e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
     }
+    frameProcessor.setInputFrameInfo(new FrameInfo(decodedWidth, decodedHeight));
 
     decoder =
         decoderFactory.createForVideoDecoding(