Use encoder output format for configuring the Encoder.

This CL implements fixing the input format to the encoder spec. Fixed
parameters include:
- MIME type
- Profile & level
- Resolution
- frame rate, and
- bitrate

PiperOrigin-RevId: 422513738

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java

@@ -22,12 +22,16 @@ import static com.google.android.exoplayer2.util.Util.SDK_INT;
 
 
 import android.annotation.SuppressLint;
 import android.annotation.SuppressLint;
 import android.media.MediaCodec;
 import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaFormat;
 import android.media.MediaFormat;
+import android.util.Pair;
 import android.view.Surface;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
 import com.google.android.exoplayer2.util.MediaFormatUtil;
 import com.google.android.exoplayer2.util.MediaFormatUtil;
+import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.TraceUtil;
 import java.io.IOException;
 import java.io.IOException;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@@ -36,6 +40,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 /* package */ final class DefaultCodecFactory
 /* package */ final class DefaultCodecFactory
     implements Codec.DecoderFactory, Codec.EncoderFactory {
     implements Codec.DecoderFactory, Codec.EncoderFactory {
 
 
+  // TODO(b/210591626) Fall back adaptively to H265 if possible.
+  private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264;
+  private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface;
+  private static final int DEFAULT_FRAME_RATE = 60;
+  private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1;
+
   @Override
   @Override
   public Codec createForAudioDecoding(Format format) throws TransformationException {
   public Codec createForAudioDecoding(Format format) throws TransformationException {
     MediaFormat mediaFormat =
     MediaFormat mediaFormat =
@@ -91,20 +101,34 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
 
   @Override
   @Override
   public Codec createForVideoEncoding(Format format) throws TransformationException {
   public Codec createForVideoEncoding(Format format) throws TransformationException {
+    checkArgument(format.sampleMimeType != null);
     checkArgument(format.width != Format.NO_VALUE);
     checkArgument(format.width != Format.NO_VALUE);
     checkArgument(format.height != Format.NO_VALUE);
     checkArgument(format.height != Format.NO_VALUE);
     // According to interface Javadoc, format.rotationDegrees should be 0. The video should always
     // According to interface Javadoc, format.rotationDegrees should be 0. The video should always
     // be in landscape orientation.
     // be in landscape orientation.
     checkArgument(format.height < format.width);
     checkArgument(format.height < format.width);
     checkArgument(format.rotationDegrees == 0);
     checkArgument(format.rotationDegrees == 0);
+    // Checking again to silence null checker warning.
+    checkNotNull(format.sampleMimeType);
+    format = getVideoEncoderSupportedFormat(format);
 
 
     MediaFormat mediaFormat =
     MediaFormat mediaFormat =
         MediaFormat.createVideoFormat(
         MediaFormat.createVideoFormat(
             checkNotNull(format.sampleMimeType), format.width, format.height);
             checkNotNull(format.sampleMimeType), format.width, format.height);
-    mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
-    mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
-    mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
-    mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000);
+    mediaFormat.setFloat(MediaFormat.KEY_FRAME_RATE, format.frameRate);
+    mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate);
+
+    @Nullable
+    Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
+    if (codecProfileAndLevel != null) {
+      mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfileAndLevel.first);
+      if (SDK_INT >= 23) {
+        mediaFormat.setInteger(MediaFormat.KEY_LEVEL, codecProfileAndLevel.second);
+      }
+    }
+
+    mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, DEFAULT_COLOR_FORMAT);
+    mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL_SECS);
 
 
     return createCodec(
     return createCodec(
         format,
         format,
@@ -168,6 +192,85 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     TraceUtil.endSection();
     TraceUtil.endSection();
   }
   }
 
 
+  @RequiresNonNull("#1.sampleMimeType")
+  private static Format getVideoEncoderSupportedFormat(Format requestedFormat)
+      throws TransformationException {
+    String mimeType = requestedFormat.sampleMimeType;
+    Format.Builder formatBuilder = requestedFormat.buildUpon();
+
+    // TODO(b/210591626) Implement encoder filtering.
+    if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+      mimeType = DEFAULT_FALLBACK_MIME_TYPE;
+      if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+        throw createTransformationException(
+            new IllegalArgumentException(
+                "No encoder is found for requested MIME type " + requestedFormat.sampleMimeType),
+            requestedFormat,
+            /* isVideo= */ true,
+            /* isDecoder= */ false,
+            /* mediaCodecName= */ null);
+      }
+    }
+
+    formatBuilder.setSampleMimeType(mimeType);
+    MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0);
+
+    int width = requestedFormat.width;
+    int height = requestedFormat.height;
+    @Nullable
+    Pair<Integer, Integer> encoderSupportedResolution =
+        EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height);
+    if (encoderSupportedResolution == null) {
+      throw createTransformationException(
+          new IllegalArgumentException(
+              "Cannot find fallback resolution for resolution " + width + " x " + height),
+          requestedFormat,
+          /* isVideo= */ true,
+          /* isDecoder= */ false,
+          /* mediaCodecName= */ null);
+    }
+    width = encoderSupportedResolution.first;
+    height = encoderSupportedResolution.second;
+    formatBuilder.setWidth(width).setHeight(height);
+
+    // The frameRate does not affect the resulting frame rate. It affects the encoder's rate control
+    // algorithm. Setting it too high may lead to video quality degradation.
+    float frameRate =
+        requestedFormat.frameRate != Format.NO_VALUE
+            ? requestedFormat.frameRate
+            : DEFAULT_FRAME_RATE;
+    int bitrate =
+        EncoderUtil.getClosestSupportedBitrate(
+            encoderInfo,
+            mimeType,
+            /* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE
+                ? requestedFormat.averageBitrate
+                : getSuggestedBitrate(width, height, frameRate));
+    formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate);
+
+    @Nullable
+    Pair<Integer, Integer> profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat);
+    if (profileLevel == null
+        // Transcoding to another MIME type.
+        || !requestedFormat.sampleMimeType.equals(mimeType)
+        || !EncoderUtil.isProfileLevelSupported(
+            encoderInfo,
+            mimeType,
+            /* profile= */ profileLevel.first,
+            /* level= */ profileLevel.second)) {
+      formatBuilder.setCodecs(null);
+    }
+
+    return formatBuilder.build();
+  }
+
+  /** Computes the video bit rate using the Kush Gauge. */
+  private static int getSuggestedBitrate(int width, int height, float frameRate) {
+    // TODO(b/210591626) Implement bitrate estimation.
+    // 1080p30 -> 6.2Mbps, 720p30 -> 2.7Mbps.
+    return (int) (width * height * frameRate * 0.1);
+  }
+
   private static TransformationException createTransformationException(
   private static TransformationException createTransformationException(
       Exception cause,
       Exception cause,
       Format format,
       Format format,

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import static java.lang.Math.round;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaCodecList;
+import android.util.Pair;
+import androidx.annotation.Nullable;
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Utility methods for {@link MediaCodec} encoders. */
+public final class EncoderUtil {
+
+  private static final List<MediaCodecInfo> encoders = new ArrayList<>();
+
+  /**
+   * Returns a list of {@link MediaCodecInfo encoders} that support the given {@code mimeType}, or
+   * an empty list if there is none.
+   */
+  public static ImmutableList<MediaCodecInfo> getSupportedEncoders(String mimeType) {
+    maybePopulateEncoderInfos();
+
+    ImmutableList.Builder<MediaCodecInfo> availableEncoders = new ImmutableList.Builder<>();
+    for (int i = 0; i < encoders.size(); i++) {
+      MediaCodecInfo encoderInfo = encoders.get(i);
+      String[] supportedMimeTypes = encoderInfo.getSupportedTypes();
+      for (String supportedMimeType : supportedMimeTypes) {
+        if (Ascii.equalsIgnoreCase(supportedMimeType, mimeType)) {
+          availableEncoders.add(encoderInfo);
+        }
+      }
+    }
+    return availableEncoders.build();
+  }
+
+  /**
+   * Finds the {@link MediaCodecInfo encoder}'s closest supported resolution from the given
+   * resolution.
+   *
+   * <p>The input resolution is returned, if it is supported by the {@link MediaCodecInfo encoder}.
+   *
+   * <p>The resolution will be clamped to the {@link MediaCodecInfo encoder}'s range of supported
+   * resolutions, and adjusted to the {@link MediaCodecInfo encoder}'s size alignment. The
+   * adjustment process takes into account the original aspect ratio. But the fixed resolution may
+   * not preserve the original aspect ratio, depending on the encoder's required size alignment.
+   *
+   * @param encoderInfo The {@link MediaCodecInfo} of the encoder.
+   * @param mimeType The output MIME type.
+   * @param width The original width.
+   * @param height The original height.
+   * @return A {@link Pair} of width and height, or {@code null} if unable to find a fix.
+   */
+  @Nullable
+  public static Pair<Integer, Integer> getClosestSupportedResolution(
+      MediaCodecInfo encoderInfo, String mimeType, int width, int height) {
+    MediaCodecInfo.VideoCapabilities videoEncoderCapabilities =
+        encoderInfo.getCapabilitiesForType(mimeType).getVideoCapabilities();
+
+    if (videoEncoderCapabilities.isSizeSupported(width, height)) {
+      return Pair.create(width, height);
+    }
+
+    // Fix frame being too wide or too tall.
+    int adjustedHeight = videoEncoderCapabilities.getSupportedHeights().clamp(height);
+    if (adjustedHeight != height) {
+      width = (int) round((double) width * adjustedHeight / height);
+      height = adjustedHeight;
+    }
+
+    int adjustedWidth = videoEncoderCapabilities.getSupportedWidths().clamp(width);
+    if (adjustedWidth != width) {
+      height = (int) round((double) height * adjustedWidth / width);
+      width = adjustedWidth;
+    }
+
+    // Fix pixel alignment.
+    width = alignResolution(width, videoEncoderCapabilities.getWidthAlignment());
+    height = alignResolution(height, videoEncoderCapabilities.getHeightAlignment());
+
+    return videoEncoderCapabilities.isSizeSupported(width, height)
+        ? Pair.create(width, height)
+        : null;
+  }
+
+  /** Returns whether the {@link MediaCodecInfo encoder} supports the given profile and level. */
+  public static boolean isProfileLevelSupported(
+      MediaCodecInfo encoderInfo, String mimeType, int profile, int level) {
+    MediaCodecInfo.CodecProfileLevel[] profileLevels =
+        encoderInfo.getCapabilitiesForType(mimeType).profileLevels;
+
+    for (MediaCodecInfo.CodecProfileLevel profileLevel : profileLevels) {
+      if (profileLevel.profile == profile && profileLevel.level == level) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Finds the {@link MediaCodecInfo encoder}'s closest supported bitrate from the given bitrate.
+   */
+  public static int getClosestSupportedBitrate(
+      MediaCodecInfo encoderInfo, String mimeType, int bitrate) {
+    return encoderInfo
+        .getCapabilitiesForType(mimeType)
+        .getVideoCapabilities()
+        .getBitrateRange()
+        .clamp(bitrate);
+  }
+
+  /**
+   * Align to the closest resolution that respects the encoder's supported alignment.
+   *
+   * <p>For example, size 35 will be aligned to 32 if the alignment is 16, and size 45 will be
+   * aligned to 48.
+   */
+  private static int alignResolution(int size, int alignment) {
+    return alignment * Math.round((float) size / alignment);
+  }
+
+  private static synchronized void maybePopulateEncoderInfos() {
+    if (encoders.isEmpty()) {
+      MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS);
+      MediaCodecInfo[] allCodecInfos = mediaCodecList.getCodecInfos();
+
+      for (MediaCodecInfo mediaCodecInfo : allCodecInfos) {
+        if (!mediaCodecInfo.isEncoder()) {
+          continue;
+        }
+        encoders.add(mediaCodecInfo);
+      }
+    }
+  }
+
+  private EncoderUtil() {}
+}