Move audio MIME type fallback away to ATSP

PiperOrigin-RevId: 491660842

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java

@@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.List;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -472,9 +471,8 @@ public final class AndroidTestUtil {
     }
 
     @Override
-    public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
-        throws TransformationException {
-      return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
+    public Codec createForAudioEncoding(Format format) throws TransformationException {
+      return encoderFactory.createForAudioEncoding(format);
     }
 
     @Override

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java

@@ -35,7 +35,6 @@ import com.google.common.base.Ascii;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.io.File;
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeoutException;
@@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner {
     }
 
     @Override
-    public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
-        throws TransformationException {
-      Codec audioEncoder = encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
+    public Codec createForAudioEncoding(Format format) throws TransformationException {
+      Codec audioEncoder = encoderFactory.createForAudioEncoding(format);
       audioEncoderName = audioEncoder.getName();
       return audioEncoder;
     }

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java

@@ -26,7 +26,6 @@ import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.MediaItem;
-import java.util.List;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -140,9 +139,8 @@ public class TransformerEndToEndTest {
     }
 
     @Override
-    public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
-        throws TransformationException {
-      return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
+    public Codec createForAudioEncoding(Format format) throws TransformationException {
+      return encoderFactory.createForAudioEncoding(format);
     }
 
     @Override

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.util.Util;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
+import java.util.List;
 import org.checkerframework.dataflow.qual.Pure;
 
 /**
@@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure;
 
     audioProcessingPipeline.flush();
 
+    String requestedMimeType =
+        transformationRequest.audioMimeType != null
+            ? transformationRequest.audioMimeType
+            : checkNotNull(inputFormat.sampleMimeType);
     Format requestedOutputFormat =
         new Format.Builder()
-            .setSampleMimeType(
-                transformationRequest.audioMimeType == null
-                    ? inputFormat.sampleMimeType
-                    : transformationRequest.audioMimeType)
+            .setSampleMimeType(requestedMimeType)
             .setSampleRate(encoderInputAudioFormat.sampleRate)
             .setChannelCount(encoderInputAudioFormat.channelCount)
             .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
             .build();
+
+    ImmutableList<String> muxerSupportedMimeTypes =
+        muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO);
+
+    // TODO(b/259570024): investigate overhauling fallback.
+    @Nullable
+    String supportedMimeType =
+        selectEncoderAndMuxerSupportedMimeType(requestedMimeType, muxerSupportedMimeTypes);
+    if (supportedMimeType == null) {
+      throw createNoSupportedMimeTypeException(requestedOutputFormat);
+    }
+
     encoder =
         encoderFactory.createForAudioEncoding(
-            requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO));
-
+            requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build());
+    checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType));
     fallbackListener.onTransformationRequestFinalized(
         createFallbackTransformationRequest(
             transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
@@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure;
     encoder.queueInputBuffer(encoderInputBuffer);
   }
 
+  @Nullable
+  private static String selectEncoderAndMuxerSupportedMimeType(
+      String requestedMimeType, List<String> muxerSupportedMimeTypes) {
+    if (!EncoderUtil.getSupportedEncoders(requestedMimeType).isEmpty()) {
+      return requestedMimeType;
+    } else {
+      // No encoder supports the requested MIME type.
+      for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) {
+        String mimeType = muxerSupportedMimeTypes.get(i);
+        if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+          return mimeType;
+        }
+      }
+    }
+    return null;
+  }
+
   @Pure
   private static TransformationRequest createFallbackTransformationRequest(
       TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java

@@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
             : null;
   }
 
+  protected static TransformationException createNoSupportedMimeTypeException(
+      Format requestedEncoderFormat) {
+    return TransformationException.createForCodec(
+        new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
+        MimeTypes.isVideo(requestedEncoderFormat.sampleMimeType),
+        /* isDecoder= */ false,
+        requestedEncoderFormat,
+        /* mediaCodecName= */ null,
+        TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
+  }
+
   @Nullable
   @Override
   public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java

@@ -22,9 +22,7 @@ import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
-import com.google.android.exoplayer2.util.MimeTypes;
 import java.nio.ByteBuffer;
-import java.util.List;
 
 /**
  * Provides a layer of abstraction for interacting with decoders and encoders.
@@ -70,19 +68,18 @@ public interface Codec {
     /**
      * Returns a {@link Codec} for audio encoding.
      *
-     * <p>This method must validate that the {@link Codec} is configured to produce one of the
-     * {@code allowedMimeTypes}. The {@linkplain Format#sampleMimeType sample MIME type} given in
-     * {@code format} is not necessarily allowed.
+     * <p>The caller should ensure the {@linkplain Format#sampleMimeType MIME type} is supported on
+     * the device before calling this method.
      *
      * @param format The {@link Format} (of the output data) used to determine the underlying
-     *     encoder and its configuration values.
-     * @param allowedMimeTypes The non-empty list of allowed output sample {@linkplain MimeTypes
-     *     MIME types}.
-     * @return A {@link Codec} for audio encoding.
+     *     encoder and its configuration values. {@link Format#sampleMimeType}, {@link
+     *     Format#sampleRate}, {@link Format#channelCount} and {@link Format#bitrate} are set to
+     *     those of the desired output video format.
+     * @return A {@link Codec} for encoding audio to the requested {@link Format#sampleMimeType MIME
+     *     type}.
      * @throws TransformationException If no suitable {@link Codec} can be created.
      */
-    Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
-        throws TransformationException;
+    Codec createForAudioEncoding(Format format) throws TransformationException;
 
     /**
      * Returns a {@link Codec} for video encoding.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java

@@ -172,23 +172,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
   }
 
   @Override
-  public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
-      throws TransformationException {
+  public DefaultCodec createForAudioEncoding(Format format) throws TransformationException {
     // TODO(b/210591626) Add encoder selection for audio.
-    checkArgument(!allowedMimeTypes.isEmpty());
     checkNotNull(format.sampleMimeType);
-    if (!allowedMimeTypes.contains(format.sampleMimeType)) {
-      if (enableFallback) {
-        // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
-        // capabilities limitations.
-        format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
-      } else {
-        throw createTransformationException(format);
-      }
-    }
     MediaFormat mediaFormat =
         MediaFormat.createAudioFormat(
-            checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
+            format.sampleMimeType, format.sampleRate, format.channelCount);
     mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
 
     @Nullable

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java

@@ -444,9 +444,7 @@ public final class EncoderUtil {
       }
       String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes();
       for (String mimeType : supportedMimeTypes) {
-        if (MimeTypes.isVideo(mimeType)) {
-          encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
-        }
+        encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
       }
     }
     return encoderInfosBuilder.build();

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure;
           selectEncoderAndMuxerSupportedMimeType(
               requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo);
       if (supportedMimeType == null) {
-        throw TransformationException.createForCodec(
-            new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
-            /* isVideo= */ true,
-            /* isDecoder= */ false,
-            requestedEncoderFormat,
-            /* mediaCodecName= */ null,
-            TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
+        throw createNoSupportedMimeTypeException(requestedEncoderFormat);
       }
 
       encoder =

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java

@@ -453,18 +453,25 @@ public final class TransformerEndToEndTest {
   }
 
   @Test
-  public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError()
+  public void startTransformation_withAudioMuxerFormatUnsupported_completesSuccessfully()
       throws Exception {
-    Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
+    // Test succeeds because MIME type fallback is mandatory.
+    Transformer.Listener mockListener = mock(Transformer.Listener.class);
+    TransformationRequest originalTransformationRequest =
+        new TransformationRequest.Builder().build();
+    TransformationRequest fallbackTransformationRequest =
+        new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build();
+    Transformer transformer =
+        createTransformerBuilder(/* enableFallback= */ false).addListener(mockListener).build();
     MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER);
 
     transformer.startTransformation(mediaItem, outputPath);
-    TransformationException exception = TransformerTestRunner.runUntilError(transformer);
+    TransformerTestRunner.runUntilCompleted(transformer);
 
-    assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
-    assertThat(exception).hasMessageThat().contains("audio");
-    assertThat(exception.errorCode)
-        .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
+    DumpFileAsserts.assertOutput(
+        context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback"));
+    verify(mockListener)
+        .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest);
   }
 
   @Test

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java

@@ -32,7 +32,6 @@ import com.google.android.exoplayer2.util.ListenerSet;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.SurfaceInfo;
 import com.google.common.collect.ImmutableList;
-import java.util.List;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest {
     }
 
     @Override
-    public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) {
+    public Codec createForAudioEncoding(Format format) {
       throw new UnsupportedOperationException();
     }