FrameProcessor: Create PresentationFrameProcessor.

Allow apps to modify how frames are presented, via modifying resolution.

A follow-up CL will provide aspect ratio, cropping, etc.

PiperOrigin-RevId: 436963312

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java

@@ -178,14 +178,12 @@ public final class FrameProcessorChainPixelTest {
   }
 
   @Test
-  public void processData_withScaleToFitFrameProcessor_requestOutputHeight_producesExpectedOutput()
-      throws Exception {
-    String testId = "processData_withScaleToFitFrameProcessor_requestOutputHeight";
-    // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
-    //  resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
-    //  test that rotation doesn't distort the image.
+  public void
+      processData_withPresentationFrameProcessor_requestOutputHeight_producesExpectedOutput()
+          throws Exception {
+    String testId = "processData_withPresentationFrameProcessor_requestOutputHeight";
     GlFrameProcessor glFrameProcessor =
-        new ScaleToFitFrameProcessor.Builder(getApplicationContext()).setResolution(480).build();
+        new PresentationFrameProcessor.Builder(getApplicationContext()).setResolution(480).build();
     setUpAndPrepareFirstFrame(glFrameProcessor);
     Bitmap expectedBitmap =
         BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
@@ -205,9 +203,6 @@ public final class FrameProcessorChainPixelTest {
   public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
       throws Exception {
     String testId = "processData_withScaleToFitFrameProcessor_rotate45";
-    // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
-    //  resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
-    //  test that rotation doesn't distort the image.
     GlFrameProcessor glFrameProcessor =
         new ScaleToFitFrameProcessor.Builder(getApplicationContext())
             .setRotationDegrees(45)

libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java

@@ -98,7 +98,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    */
   public AdvancedFrameProcessor(Context context, Matrix transformationMatrix) {
     this.context = context;
-    this.transformationMatrix = transformationMatrix;
+    this.transformationMatrix = new Matrix(transformationMatrix);
   }
 
   @Override

libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+
+import android.content.Context;
+import android.graphics.Matrix;
+import android.util.Size;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.util.GlUtil;
+import java.io.IOException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/** Controls how a frame is viewed, by changing resolution. */
+// TODO(b/213190310): Implement crop, aspect ratio changes, etc.
+/* package */ final class PresentationFrameProcessor implements GlFrameProcessor {
+
+  /** A builder for {@link PresentationFrameProcessor} instances. */
+  public static final class Builder {
+    // Mandatory field.
+    private final Context context;
+
+    // Optional field.
+    private int outputHeight;
+
+    /**
+     * Creates a builder with default values.
+     *
+     * @param context The {@link Context}.
+     */
+    public Builder(Context context) {
+      this.context = context;
+      outputHeight = C.LENGTH_UNSET;
+    }
+
+    /**
+     * Sets the output resolution using the output height.
+     *
+     * <p>The default value {@link C#LENGTH_UNSET} corresponds to using the same height as the
+     * input. Output width of the displayed frame will scale to preserve the frame's aspect ratio
+     * after other transformations.
+     *
+     * <p>For example, a 1920x1440 frame can be scaled to 640x480 by calling setResolution(480).
+     *
+     * @param outputHeight The output height of the displayed frame, in pixels.
+     * @return This builder.
+     */
+    public Builder setResolution(int outputHeight) {
+      this.outputHeight = outputHeight;
+      return this;
+    }
+
+    public PresentationFrameProcessor build() {
+      return new PresentationFrameProcessor(context, outputHeight);
+    }
+  }
+
+  static {
+    GlUtil.glAssertionsEnabled = true;
+  }
+
+  private final Context context;
+  private final int requestedHeight;
+
+  private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
+  private int inputWidth;
+  private int inputHeight;
+  private int outputHeight;
+  private int outputRotationDegrees;
+  private @MonotonicNonNull Matrix transformationMatrix;
+
+  /**
+   * Creates a new instance.
+   *
+   * @param context The {@link Context}.
+   * @param requestedHeight The height of the output frame, in pixels.
+   */
+  private PresentationFrameProcessor(Context context, int requestedHeight) {
+    this.context = context;
+    this.requestedHeight = requestedHeight;
+
+    inputWidth = C.LENGTH_UNSET;
+    inputHeight = C.LENGTH_UNSET;
+    outputHeight = C.LENGTH_UNSET;
+    outputRotationDegrees = C.LENGTH_UNSET;
+  }
+
+  /**
+   * Returns {@link Format#rotationDegrees} for the output frame.
+   *
+   * <p>Return values may be {@code 0} or {@code 90} degrees.
+   *
+   * <p>This method can only be called after {@link #configureOutputSize(int, int)}.
+   */
+  public int getOutputRotationDegrees() {
+    checkState(outputRotationDegrees != C.LENGTH_UNSET);
+    return outputRotationDegrees;
+  }
+
+  /**
+   * Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame.
+   *
+   * <p>The {@code PresentationFrameProcessor} should only be used if this returns true.
+   *
+   * <p>This method can only be called after {@link #configureOutputSize(int, int)}.
+   */
+  public boolean shouldProcess() {
+    checkStateNotNull(transformationMatrix);
+    return inputHeight != outputHeight || !transformationMatrix.isIdentity();
+  }
+
+  @Override
+  public Size configureOutputSize(int inputWidth, int inputHeight) {
+    this.inputWidth = inputWidth;
+    this.inputHeight = inputHeight;
+    transformationMatrix = new Matrix();
+
+    int displayWidth = inputWidth;
+    int displayHeight = inputHeight;
+
+    // Scale width and height to desired requestedHeight, preserving aspect ratio.
+    if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) {
+      displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight);
+      displayHeight = requestedHeight;
+    }
+
+    int outputWidth;
+    // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
+    // frame before encoding, so the encoded frame's width >= height, and set
+    // outputRotationDegrees to ensure the frame is displayed in the correct orientation.
+    if (displayHeight > displayWidth) {
+      outputRotationDegrees = 90;
+      outputWidth = displayHeight;
+      outputHeight = displayWidth;
+      // TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in
+      //  a later GlFrameProcessor.
+      transformationMatrix.postRotate(outputRotationDegrees);
+    } else {
+      outputRotationDegrees = 0;
+      outputWidth = displayWidth;
+      outputHeight = displayHeight;
+    }
+
+    return new Size(outputWidth, outputHeight);
+  }
+
+  @Override
+  public void initialize(int inputTexId) throws IOException {
+    checkStateNotNull(transformationMatrix);
+    advancedFrameProcessor = new AdvancedFrameProcessor(context, transformationMatrix);
+    advancedFrameProcessor.configureOutputSize(inputWidth, inputHeight);
+    advancedFrameProcessor.initialize(inputTexId);
+  }
+
+  @Override
+  public void updateProgramAndDraw(long presentationTimeNs) {
+    checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs);
+  }
+
+  @Override
+  public void release() {
+    if (advancedFrameProcessor != null) {
+      advancedFrameProcessor.release();
+    }
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java

@@ -140,7 +140,6 @@ public final class TransformationRequest {
      * @return This builder.
      */
     public Builder setResolution(int outputHeight) {
-      // TODO(b/201293185): Restructure to input a Presentation class.
       this.outputHeight = outputHeight;
       return this;
     }

libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java

@@ -74,15 +74,19 @@ import org.checkerframework.dataflow.qual.Pure;
         new ScaleToFitFrameProcessor.Builder(context)
             .setScale(transformationRequest.scaleX, transformationRequest.scaleY)
             .setRotationDegrees(transformationRequest.rotationDegrees)
+            .build();
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(context)
             .setResolution(transformationRequest.outputHeight)
             .build();
     // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
-    ImmutableList<GlFrameProcessor> frameProcessors = ImmutableList.of(scaleToFitFrameProcessor);
+    ImmutableList<GlFrameProcessor> frameProcessors =
+        ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor);
     List<Size> frameProcessorSizes =
         FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors);
     Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes);
     // TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor.
-    outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees();
+    outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();
 
     Format requestedEncoderFormat =
         new Format.Builder()
@@ -109,6 +113,7 @@ import org.checkerframework.dataflow.qual.Pure;
         || inputFormat.height != encoderSupportedFormat.height
         || inputFormat.width != encoderSupportedFormat.width
         || scaleToFitFrameProcessor.shouldProcess()
+        || presentationFrameProcessor.shouldProcess()
         || shouldAlwaysUseFrameProcessorChain()) {
       // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
       //  has to change due to encoder fallback or append another GlFrameProcessor.

libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import android.util.Size;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Unit tests for {@link PresentationFrameProcessor}.
+ *
+ * <p>See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link
+ * AdvancedFrameProcessor} given a transformation matrix.
+ */
+@RunWith(AndroidJUnit4.class)
+public final class PresentationFrameProcessorTest {
+  @Test
+  public void configureOutputSize_noEditsLandscape_leavesFramesUnchanged() {
+    int inputWidth = 200;
+    int inputHeight = 150;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext()).build();
+
+    Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
+    assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
+    assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
+  }
+
+  @Test
+  public void configureOutputSize_noEditsSquare_leavesFramesUnchanged() {
+    int inputWidth = 150;
+    int inputHeight = 150;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext()).build();
+
+    Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    assertThat(presentationFrameProcessor.shouldProcess()).isFalse();
+    assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
+    assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
+  }
+
+  @Test
+  public void configureOutputSize_noEditsPortrait_flipsOrientation() {
+    int inputWidth = 150;
+    int inputHeight = 200;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext()).build();
+
+    Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
+    assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
+    assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
+    assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
+  }
+
+  @Test
+  public void configureOutputSize_setResolution_changesDimensions() {
+    int inputWidth = 200;
+    int inputHeight = 150;
+    int requestedHeight = 300;
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext())
+            .setResolution(requestedHeight)
+            .build();
+
+    Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight);
+
+    assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
+    assertThat(presentationFrameProcessor.shouldProcess()).isTrue();
+    assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
+    assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
+  }
+
+  @Test
+  public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
+    PresentationFrameProcessor presentationFrameProcessor =
+        new PresentationFrameProcessor.Builder(getApplicationContext()).build();
+
+    // configureOutputSize not called before initialize.
+    assertThrows(IllegalStateException.class, presentationFrameProcessor::getOutputRotationDegrees);
+  }
+}

libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java

@@ -34,7 +34,7 @@ import org.junit.runner.RunWith;
 public final class ScaleToFitFrameProcessorTest {
 
   @Test
-  public void configureOutputDimensions_noEdits_producesExpectedOutput() {
+  public void configureOutputSize_noEdits_leavesFramesUnchanged() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
@@ -53,23 +52,14 @@ public final class ScaleToFitFrameProcessorTest {
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
         new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build();
 
-    // configureOutputDimensions not called before initialize.
+    // configureOutputSize not called before initialize.
     assertThrows(
         IllegalStateException.class,
         () -> scaleToFitFrameProcessor.initialize(/* inputTexId= */ 0));
   }
 
   @Test
-  public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
-    ScaleToFitFrameProcessor scaleToFitFrameProcessor =
-        new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build();
-
-    // configureOutputDimensions not called before initialize.
-    assertThrows(IllegalStateException.class, scaleToFitFrameProcessor::getOutputRotationDegrees);
-  }
-
-  @Test
-  public void configureOutputDimensions_scaleNarrow_producesExpectedOutput() {
+  public void configureOutputSize_scaleNarrow_decreasesWidth() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -79,14 +69,13 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
-    assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
-    assertThat(outputSize.getHeight()).isEqualTo(Math.round(inputWidth * .5f));
+    assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f));
+    assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
 
   @Test
-  public void configureOutputDimensions_scaleWide_producesExpectedOutput() {
+  public void configureOutputSize_scaleWide_increasesWidth() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -96,14 +85,13 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2);
     assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
   }
 
   @Test
-  public void configureOutputDimensions_scaleTall_producesExpectedOutput() {
+  public void configureOutputDimensions_scaleTall_increasesHeight() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -113,14 +101,13 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
-    assertThat(outputSize.getWidth()).isEqualTo(inputHeight * 2);
-    assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
+    assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
+    assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2);
   }
 
   @Test
-  public void configureOutputDimensions_rotate90_producesExpectedOutput() {
+  public void configureOutputSize_rotate90_swapsDimensions() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -130,14 +117,13 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
-    assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
-    assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
+    assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
+    assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
   }
 
   @Test
-  public void configureOutputDimensions_rotate45_producesExpectedOutput() {
+  public void configureOutputSize_rotate45_changesDimensions() {
     int inputWidth = 200;
     int inputHeight = 150;
     ScaleToFitFrameProcessor scaleToFitFrameProcessor =
@@ -148,27 +134,8 @@ public final class ScaleToFitFrameProcessorTest {
 
     Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
 
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
     assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
     assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight);
     assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight);
   }
-
-  @Test
-  public void configureOutputDimensions_setResolution_producesExpectedOutput() {
-    int inputWidth = 200;
-    int inputHeight = 150;
-    int requestedHeight = 300;
-    ScaleToFitFrameProcessor scaleToFitFrameProcessor =
-        new ScaleToFitFrameProcessor.Builder(getApplicationContext())
-            .setResolution(requestedHeight)
-            .build();
-
-    Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight);
-
-    assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
-    assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
-    assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
-    assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
-  }
 }