FrameProcessor: Replace SurfaceInfo.Provider with setter.

The FinalMatrixTransformationProcessorWrapper ensures that the
surface is only replaced when it is not being rendered to and vice
versa.

PiperOrigin-RevId: 458007639
(cherry picked from commit e5527a8a)

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java

@@ -357,6 +357,16 @@ public final class GlEffectsFrameProcessorPixelTest {
                   context,
                   new FrameProcessor.Listener() {
                     @Override
+                    public void onOutputSizeChanged(int width, int height) {
+                      outputImageReader =
+                          ImageReader.newInstance(
+                              width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
+                      checkNotNull(glEffectsFrameProcessor)
+                          .setOutputSurfaceInfo(
+                              new SurfaceInfo(outputImageReader.getSurface(), width, height));
+                    }
+
+                    @Override
                     public void onFrameProcessingError(FrameProcessingException exception) {
                       frameProcessingException.set(exception);
                     }
@@ -368,16 +378,6 @@ public final class GlEffectsFrameProcessorPixelTest {
                   },
                   /* streamOffsetUs= */ 0L,
                   effects,
-                  /* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
-                    outputImageReader =
-                        ImageReader.newInstance(
-                            requestedWidth,
-                            requestedHeight,
-                            PixelFormat.RGBA_8888,
-                            /* maxImages= */ 1);
-                    return new SurfaceInfo(
-                        outputImageReader.getSurface(), requestedWidth, requestedHeight);
-                  },
                   Transformer.DebugViewProvider.NONE,
                   /* enableExperimentalHdrEditing= */ false));
       glEffectsFrameProcessor.setInputFrameInfo(

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java

@@ -34,6 +34,7 @@ import androidx.annotation.WorkerThread;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Log;
+import com.google.android.exoplayer2.util.Util;
 import com.google.common.collect.ImmutableList;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -57,7 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final ImmutableList<GlMatrixTransformation> matrixTransformations;
   private final EGLDisplay eglDisplay;
   private final EGLContext eglContext;
-  private final SurfaceInfo.Provider outputSurfaceProvider;
   private final long streamOffsetUs;
   private final Transformer.DebugViewProvider debugViewProvider;
   private final FrameProcessor.Listener frameProcessorListener;
@@ -66,17 +66,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private int inputWidth;
   private int inputHeight;
   @Nullable private MatrixTransformationProcessor matrixTransformationProcessor;
-  @Nullable private SurfaceInfo outputSurfaceInfo;
-  @Nullable private EGLSurface outputEglSurface;
   @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
   private @MonotonicNonNull Listener listener;
+  private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
+
+  @GuardedBy("this")
+  @Nullable
+  private SurfaceInfo outputSurfaceInfo;
+
+  @GuardedBy("this")
+  @Nullable
+  private EGLSurface outputEglSurface;
 
   public FinalMatrixTransformationProcessorWrapper(
       Context context,
       EGLDisplay eglDisplay,
       EGLContext eglContext,
       ImmutableList<GlMatrixTransformation> matrixTransformations,
-      SurfaceInfo.Provider outputSurfaceProvider,
       long streamOffsetUs,
       FrameProcessor.Listener frameProcessorListener,
       Transformer.DebugViewProvider debugViewProvider,
@@ -85,7 +91,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.matrixTransformations = matrixTransformations;
     this.eglDisplay = eglDisplay;
     this.eglContext = eglContext;
-    this.outputSurfaceProvider = outputSurfaceProvider;
     this.streamOffsetUs = streamOffsetUs;
     this.debugViewProvider = debugViewProvider;
     this.frameProcessorListener = frameProcessorListener;
@@ -107,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @Override
   public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
     try {
+      synchronized (this) {
         if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
           return false;
         }
@@ -129,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             outputEglSurface,
             /* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
         EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
+      }
     } catch (FrameProcessingException | GlUtil.GlException e) {
       frameProcessorListener.onFrameProcessingError(
           FrameProcessingException.from(e, presentationTimeUs));
@@ -156,24 +163,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   @EnsuresNonNullIf(
       expression = {"outputSurfaceInfo", "outputEglSurface", "matrixTransformationProcessor"},
       result = true)
-  private boolean ensureConfigured(int inputWidth, int inputHeight)
+  private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
       throws FrameProcessingException, GlUtil.GlException {
-    if (inputWidth == this.inputWidth
-        && inputHeight == this.inputHeight
-        && outputSurfaceInfo != null
-        && outputEglSurface != null
-        && matrixTransformationProcessor != null) {
-      return true;
-    }
 
+    if (this.inputWidth != inputWidth
+        || this.inputHeight != inputHeight
+        || this.outputSizeBeforeSurfaceTransformation == null) {
       this.inputWidth = inputWidth;
       this.inputHeight = inputHeight;
-    Size requestedOutputSize =
+      Size outputSizeBeforeSurfaceTransformation =
           MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations);
-    @Nullable
-    SurfaceInfo outputSurfaceInfo =
-        outputSurfaceProvider.getSurfaceInfo(
-            requestedOutputSize.getWidth(), requestedOutputSize.getHeight());
+      if (!Util.areEqual(
+          this.outputSizeBeforeSurfaceTransformation, outputSizeBeforeSurfaceTransformation)) {
+        this.outputSizeBeforeSurfaceTransformation = outputSizeBeforeSurfaceTransformation;
+        frameProcessorListener.onOutputSizeChanged(
+            outputSizeBeforeSurfaceTransformation.getWidth(),
+            outputSizeBeforeSurfaceTransformation.getHeight());
+      }
+    }
+
     if (outputSurfaceInfo == null) {
       if (matrixTransformationProcessor != null) {
         matrixTransformationProcessor.release();
@@ -182,13 +190,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       outputEglSurface = null;
       return false;
     }
-    if (outputSurfaceInfo == this.outputSurfaceInfo
-        && outputEglSurface != null
-        && matrixTransformationProcessor != null) {
-      return true;
-    }
 
-    EGLSurface outputEglSurface;
+    SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
+    @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
+    if (outputEglSurface == null) { // This means that outputSurfaceInfo changed.
       if (enableExperimentalHdrEditing) {
         // TODO(b/227624622): Don't assume BT.2020 PQ input/output.
         outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
@@ -205,9 +210,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             new SurfaceViewWrapper(
                 eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
       }
+      if (matrixTransformationProcessor != null) {
+        matrixTransformationProcessor.release();
+        matrixTransformationProcessor = null;
+      }
+    }
 
+    if (matrixTransformationProcessor == null) {
       matrixTransformationProcessor =
-        createMatrixTransformationProcessorForOutputSurface(requestedOutputSize, outputSurfaceInfo);
+          createMatrixTransformationProcessorForOutputSurface(outputSurfaceInfo);
+    }
 
     this.outputSurfaceInfo = outputSurfaceInfo;
     this.outputEglSurface = outputEglSurface;
@@ -215,7 +227,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   private MatrixTransformationProcessor createMatrixTransformationProcessorForOutputSurface(
-      Size requestedOutputSize, SurfaceInfo outputSurfaceInfo) throws FrameProcessingException {
+      SurfaceInfo outputSurfaceInfo) throws FrameProcessingException {
     ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
         new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations);
     if (outputSurfaceInfo.orientationDegrees != 0) {
@@ -224,12 +236,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               .setRotationDegrees(outputSurfaceInfo.orientationDegrees)
               .build());
     }
-    if (outputSurfaceInfo.width != requestedOutputSize.getWidth()
-        || outputSurfaceInfo.height != requestedOutputSize.getHeight()) {
     matrixTransformationListBuilder.add(
         Presentation.createForWidthAndHeight(
             outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
-    }
 
     MatrixTransformationProcessor matrixTransformationProcessor =
         new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
@@ -258,6 +267,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
+    if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
+      this.outputSurfaceInfo = outputSurfaceInfo;
+      this.outputEglSurface = null;
+    }
+  }
+
   /**
    * Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
    * and makes rendering a no-op if not.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java

@@ -16,6 +16,7 @@
 package com.google.android.exoplayer2.transformer;
 
 import android.view.Surface;
+import androidx.annotation.Nullable;
 
 /** Interface for a frame processor that applies changes to individual video frames. */
 /* package */ interface FrameProcessor {
@@ -27,6 +28,14 @@ import android.view.Surface;
   interface Listener {
 
     /**
+     * Called when the output size after applying the final effect changes.
+     *
+     * <p>The output size after applying the final effect can differ from the size specified using
+     * {@link #setOutputSurfaceInfo(SurfaceInfo)}.
+     */
+    void onOutputSizeChanged(int width, int height);
+
+    /**
      * Called when an exception occurs during asynchronous frame processing.
      *
      * <p>If an error occurred, consuming and producing further frames will not work as expected and
@@ -69,6 +78,23 @@ import android.view.Surface;
   int getPendingInputFrameCount();
 
   /**
+   * Sets the output surface and supporting information.
+   *
+   * <p>The new output {@link SurfaceInfo} is applied from the next output frame rendered onwards.
+   * If the output {@link SurfaceInfo} is {@code null}, the {@code FrameProcessor} will stop
+   * rendering and resume rendering pending frames once a non-null {@link SurfaceInfo} is set.
+   *
+   * <p>If the dimensions given in {@link SurfaceInfo} do not match the {@linkplain
+   * Listener#onOutputSizeChanged(int,int) output size after applying the final effect} the frames
+   * are resized before rendering to the surface and letter/pillar-boxing is applied.
+   *
+   * <p>The caller is responsible for tracking the lifecycle of the {@link SurfaceInfo#surface}
+   * including calling this method with a new surface if it is destroyed. When this method returns,
+   * the previous output surface is no longer being used and can safely be released by the caller.
+   */
+  void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo);
+
+  /**
    * Informs the {@code FrameProcessor} that no further input frames should be accepted.
    *
    * @throws IllegalStateException If called more than once.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SurfaceInfo.java

@@ -55,16 +55,27 @@ import androidx.annotation.Nullable;
     this.orientationDegrees = orientationDegrees;
   }
 
-  /** A provider for a {@link SurfaceInfo} instance. */
-  public interface Provider {
-    /**
-     * Provides a {@linkplain SurfaceInfo surface} for the requested dimensions.
-     *
-     * <p>The dimensions given in the provided {@link SurfaceInfo} may differ from the requested
-     * dimensions. It is up to the caller to transform frames from the requested dimensions to the
-     * provided dimensions before rendering them to the {@link SurfaceInfo#surface}.
-     */
-    @Nullable
-    SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight);
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof SurfaceInfo)) {
+      return false;
+    }
+    SurfaceInfo that = (SurfaceInfo) o;
+    return width == that.width
+        && height == that.height
+        && orientationDegrees == that.orientationDegrees
+        && surface.equals(that.surface);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = surface.hashCode();
+    result = 31 * result + width;
+    result = 31 * result + height;
+    result = 31 * result + orientationDegrees;
+    return result;
   }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -95,8 +95,7 @@ import org.checkerframework.dataflow.qual.Pure;
             inputFormat,
             allowedOutputMimeTypes,
             transformationRequest,
-            fallbackListener,
-            asyncErrorListener);
+            fallbackListener);
 
     try {
       frameProcessor =
@@ -104,6 +103,16 @@ import org.checkerframework.dataflow.qual.Pure;
               context,
               new FrameProcessor.Listener() {
                 @Override
+                public void onOutputSizeChanged(int width, int height) {
+                  try {
+                    checkNotNull(frameProcessor)
+                        .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
+                  } catch (TransformationException exception) {
+                    asyncErrorListener.onTransformationException(exception);
+                  }
+                }
+
+                @Override
                 public void onFrameProcessingError(FrameProcessingException exception) {
                   asyncErrorListener.onTransformationException(
                       TransformationException.createForFrameProcessingException(
@@ -121,7 +130,6 @@ import org.checkerframework.dataflow.qual.Pure;
               },
               streamOffsetUs,
               effectsListBuilder.build(),
-              /* outputSurfaceProvider= */ encoderWrapper,
               debugViewProvider,
               transformationRequest.enableHdrEditing);
     } catch (FrameProcessingException e) {
@@ -284,14 +292,13 @@ import org.checkerframework.dataflow.qual.Pure;
    * dimensions, the same encoder is used and the provided dimensions stay fixed.
    */
   @VisibleForTesting
-  /* package */ static final class EncoderWrapper implements SurfaceInfo.Provider {
+  /* package */ static final class EncoderWrapper {
 
     private final Codec.EncoderFactory encoderFactory;
     private final Format inputFormat;
     private final List<String> allowedOutputMimeTypes;
     private final TransformationRequest transformationRequest;
     private final FallbackListener fallbackListener;
-    private final Transformer.AsyncErrorListener asyncErrorListener;
 
     private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;
 
@@ -304,20 +311,18 @@ import org.checkerframework.dataflow.qual.Pure;
         Format inputFormat,
         List<String> allowedOutputMimeTypes,
         TransformationRequest transformationRequest,
-        FallbackListener fallbackListener,
-        Transformer.AsyncErrorListener asyncErrorListener) {
+        FallbackListener fallbackListener) {
 
       this.encoderFactory = encoderFactory;
       this.inputFormat = inputFormat;
       this.allowedOutputMimeTypes = allowedOutputMimeTypes;
       this.transformationRequest = transformationRequest;
       this.fallbackListener = fallbackListener;
-      this.asyncErrorListener = asyncErrorListener;
     }
 
-    @Override
     @Nullable
-    public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) {
+    public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight)
+        throws TransformationException {
       if (releaseEncoder) {
         return null;
       }
@@ -349,13 +354,8 @@ import org.checkerframework.dataflow.qual.Pure;
                       : inputFormat.sampleMimeType)
               .build();
 
-      try {
       encoder =
           encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
-      } catch (TransformationException e) {
-        asyncErrorListener.onTransformationException(e);
-        return null;
-      }
       Format encoderSupportedFormat = encoder.getConfigurationFormat();
       fallbackListener.onTransformationRequestFinalized(
           createFallbackTransformationRequest(

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java

@@ -50,8 +50,7 @@ public final class VideoEncoderWrapperTest {
           /* inputFormat= */ new Format.Builder().build(),
           /* allowedOutputMimeTypes= */ ImmutableList.of(),
           emptyTransformationRequest,
-          fallbackListener,
-          mock(Transformer.AsyncErrorListener.class));
+          fallbackListener);
 
   @Before
   public void registerTrack() {
@@ -59,7 +58,7 @@ public final class VideoEncoderWrapperTest {
   }
 
   @Test
-  public void getSurfaceInfo_landscape_leavesOrientationUnchanged() {
+  public void getSurfaceInfo_landscape_leavesOrientationUnchanged() throws Exception {
     int inputWidth = 200;
     int inputHeight = 150;
 
@@ -71,7 +70,7 @@ public final class VideoEncoderWrapperTest {
   }
 
   @Test
-  public void getSurfaceInfo_square_leavesOrientationUnchanged() {
+  public void getSurfaceInfo_square_leavesOrientationUnchanged() throws Exception {
     int inputWidth = 150;
     int inputHeight = 150;
 
@@ -83,7 +82,7 @@ public final class VideoEncoderWrapperTest {
   }
 
   @Test
-  public void getSurfaceInfo_portrait_flipsOrientation() {
+  public void getSurfaceInfo_portrait_flipsOrientation() throws Exception {
     int inputWidth = 150;
     int inputHeight = 200;
 
@@ -95,7 +94,8 @@ public final class VideoEncoderWrapperTest {
   }
 
   @Test
-  public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() {
+  public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution()
+      throws TransformationException {
     int inputWidth = 200;
     int inputHeight = 150;
     int fallbackWidth = 100;