Transcode to a muxer-supported sample MIME type.

If the output sample MIME type is inferred from the input
but is not supported by the muxer, we fallback to transcoding
to a supported sample MIME type.
The audio and video renderers need to make sure not to select the PassthroughSamplePipeline for this case. Which sample MIME type
to choose is decided by the EncoderFactory.

PiperOrigin-RevId: 423272812

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.audio.SonicAudioProcessor;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.util.Util;
 import java.nio.ByteBuffer;
+import java.util.List;
 import org.checkerframework.dataflow.qual.Pure;
 
 /**
@@ -63,8 +64,9 @@ import org.checkerframework.dataflow.qual.Pure;
   public AudioSamplePipeline(
       Format inputFormat,
       TransformationRequest transformationRequest,
-      Codec.EncoderFactory encoderFactory,
       Codec.DecoderFactory decoderFactory,
+      Codec.EncoderFactory encoderFactory,
+      List<String> allowedOutputMimeTypes,
       FallbackListener fallbackListener)
       throws TransformationException {
     decoderInputBuffer =
@@ -110,7 +112,7 @@ import org.checkerframework.dataflow.qual.Pure;
             .setChannelCount(encoderInputAudioFormat.channelCount)
             .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
             .build();
-    encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat);
+    encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes);
 
     fallbackListener.onTransformationRequestFinalized(
         createFallbackRequest(

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java

@@ -30,6 +30,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
@@ -54,7 +55,7 @@ public final class Codec {
      * @param format The {@link Format} (of the input data) used to determine the underlying {@link
      *     MediaCodec} and its configuration values.
      * @return A configured and started decoder wrapper.
-     * @throws TransformationException If the underlying codec cannot be created.
+     * @throws TransformationException If no suitable codec can be created.
      */
     Codec createForAudioDecoding(Format format) throws TransformationException;
 
@@ -65,7 +66,7 @@ public final class Codec {
      *     MediaCodec} and its configuration values.
      * @param outputSurface The {@link Surface} to which the decoder output is rendered.
      * @return A configured and started decoder wrapper.
-     * @throws TransformationException If the underlying codec cannot be created.
+     * @throws TransformationException If no suitable codec can be created.
      */
     Codec createForVideoDecoding(Format format, Surface outputSurface)
         throws TransformationException;
@@ -80,25 +81,39 @@ public final class Codec {
     /**
      * Returns a {@link Codec} for audio encoding.
      *
+     * <p>This method must validate that the {@link Codec} is configured to produce one of the
+     * {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
+     * format} is not necessarily allowed.
+     *
      * @param format The {@link Format} (of the output data) used to determine the underlying {@link
      *     MediaCodec} and its configuration values.
+     * @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
+     *     types}.
      * @return A configured and started encoder wrapper.
-     * @throws TransformationException If the underlying codec cannot be created.
+     * @throws TransformationException If no suitable codec can be created.
      */
-    Codec createForAudioEncoding(Format format) throws TransformationException;
+    Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
+        throws TransformationException;
 
     /**
      * Returns a {@link Codec} for video encoding.
      *
+     * <p>This method must validate that the {@link Codec} is configured to produce one of the
+     * {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
+     * format} is not necessarily allowed.
+     *
      * @param format The {@link Format} (of the output data) used to determine the underlying {@link
      *     MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
      *     Format#width} and {@link Format#height} must be set to those of the desired output video
      *     format. {@link Format#rotationDegrees} should be 0. The video should always be in
      *     landscape orientation.
+     * @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
+     *     types}.
      * @return A configured and started encoder wrapper.
-     * @throws TransformationException If the underlying codec cannot be created.
+     * @throws TransformationException If no suitable codec can be created.
      */
-    Codec createForVideoEncoding(Format format) throws TransformationException;
+    Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes)
+        throws TransformationException;
   }
 
   // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java

@@ -34,13 +34,15 @@ import com.google.android.exoplayer2.util.MediaFormatUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import java.io.IOException;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
 /** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */
 /* package */ final class DefaultCodecFactory
     implements Codec.DecoderFactory, Codec.EncoderFactory {
+  // TODO(b/214973843): Add option to disable fallback.
 
-  // TODO(b/210591626) Fall back adaptively to H265 if possible.
+  // TODO(b/210591626): Fall back adaptively to H265 if possible.
   private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264;
   private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface;
   private static final int DEFAULT_FRAME_RATE = 60;
@@ -85,7 +87,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   }
 
   @Override
-  public Codec createForAudioEncoding(Format format) throws TransformationException {
+  public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
+      throws TransformationException {
+    checkArgument(!allowedMimeTypes.isEmpty());
+    if (!allowedMimeTypes.contains(format.sampleMimeType)) {
+      // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
+      // capabilities limitations.
+      format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
+    }
     MediaFormat mediaFormat =
         MediaFormat.createAudioFormat(
             checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
@@ -100,7 +109,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   }
 
   @Override
-  public Codec createForVideoEncoding(Format format) throws TransformationException {
+  public Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes)
+      throws TransformationException {
     checkArgument(format.width != Format.NO_VALUE);
     checkArgument(format.height != Format.NO_VALUE);
     // According to interface Javadoc, format.rotationDegrees should be 0. The video should always
@@ -108,7 +118,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     checkArgument(format.height <= format.width);
     checkArgument(format.rotationDegrees == 0);
     checkNotNull(format.sampleMimeType);
-    format = getVideoEncoderSupportedFormat(format);
+
+    checkArgument(!allowedMimeTypes.isEmpty());
+
+    format = getVideoEncoderSupportedFormat(format, allowedMimeTypes);
 
     MediaFormat mediaFormat =
         MediaFormat.createVideoFormat(
@@ -191,14 +204,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   }
 
   @RequiresNonNull("#1.sampleMimeType")
-  private static Format getVideoEncoderSupportedFormat(Format requestedFormat)
-      throws TransformationException {
+  private static Format getVideoEncoderSupportedFormat(
+      Format requestedFormat, List<String> allowedMimeTypes) throws TransformationException {
     String mimeType = requestedFormat.sampleMimeType;
     Format.Builder formatBuilder = requestedFormat.buildUpon();
 
     // TODO(b/210591626) Implement encoder filtering.
-    if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
-      mimeType = DEFAULT_FALLBACK_MIME_TYPE;
+    if (!allowedMimeTypes.contains(mimeType)
+        || EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+      mimeType =
+          allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE)
+              ? DEFAULT_FALLBACK_MIME_TYPE
+              : allowedMimeTypes.get(0);
       if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
         throw createTransformationException(
             new IllegalArgumentException(

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java

@@ -68,23 +68,18 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
       return false;
     }
     Format inputFormat = checkNotNull(formatHolder.format);
-    String sampleMimeType = checkNotNull(inputFormat.sampleMimeType);
-    if (transformationRequest.audioMimeType == null
-        && !muxerWrapper.supportsSampleMimeType(sampleMimeType)) {
-      throw TransformationException.createForMuxer(
-          new IllegalArgumentException(
-              "The output sample MIME inferred from the input format is not supported by the muxer."
-                  + " Sample MIME type: "
-                  + sampleMimeType),
-          TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
-    }
     if (shouldPassthrough(inputFormat)) {
       samplePipeline =
           new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
     } else {
       samplePipeline =
           new AudioSamplePipeline(
-              inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener);
+              inputFormat,
+              transformationRequest,
+              decoderFactory,
+              encoderFactory,
+              muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
+              fallbackListener);
     }
     return true;
   }
@@ -94,6 +89,10 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
         && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) {
       return false;
     }
+    if (transformationRequest.audioMimeType == null
+        && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
+      return false;
+    }
     if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) {
       return false;
     }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java

@@ -77,16 +77,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
     }
     Format inputFormat = checkNotNull(formatHolder.format);
-    String sampleMimeType = checkNotNull(inputFormat.sampleMimeType);
-    if (transformationRequest.videoMimeType == null
-        && !muxerWrapper.supportsSampleMimeType(sampleMimeType)) {
-      throw TransformationException.createForMuxer(
-          new IllegalArgumentException(
-              "The output sample MIME inferred from the input format is not supported by the muxer."
-                  + " Sample MIME type: "
-                  + sampleMimeType),
-          TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
-    }
     if (shouldPassthrough(inputFormat)) {
       samplePipeline =
           new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
@@ -96,8 +86,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
               context,
               inputFormat,
               transformationRequest,
-              encoderFactory,
               decoderFactory,
+              encoderFactory,
+              muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
               fallbackListener,
               debugViewProvider);
     }
@@ -112,6 +103,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) {
       return false;
     }
+    if (transformationRequest.videoMimeType == null
+        && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
+      return false;
+    }
     if (transformationRequest.outputHeight != C.LENGTH_UNSET
         && transformationRequest.outputHeight != inputFormat.height) {
       return false;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java

@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.util.Util;
+import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.dataflow.qual.Pure;
 
@@ -54,8 +55,9 @@ import org.checkerframework.dataflow.qual.Pure;
       Context context,
       Format inputFormat,
       TransformationRequest transformationRequest,
-      Codec.EncoderFactory encoderFactory,
       Codec.DecoderFactory decoderFactory,
+      Codec.EncoderFactory encoderFactory,
+      List<String> allowedOutputMimeTypes,
       FallbackListener fallbackListener,
       Transformer.DebugViewProvider debugViewProvider)
       throws TransformationException {
@@ -115,7 +117,7 @@ import org.checkerframework.dataflow.qual.Pure;
                     ? transformationRequest.videoMimeType
                     : inputFormat.sampleMimeType)
             .build();
-    encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat);
+    encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat, allowedOutputMimeTypes);
     fallbackListener.onTransformationRequestFinalized(
         createFallbackRequest(
             transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));

testdata/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump

+containerMimeType = video/mp4
+format 0:
+  sampleMimeType = audio/mp4a-latm
+  channelCount = 6
+  sampleRate = 48000
+  pcmEncoding = 2
+sample:
+  trackIndex = 0
+  dataHashCode = 1896404418
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 0
+sample:
+  trackIndex = 0
+  dataHashCode = -2134951116
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 2667
+sample:
+  trackIndex = 0
+  dataHashCode = 97556101
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 5334
+sample:
+  trackIndex = 0
+  dataHashCode = -1448980924
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 8000
+sample:
+  trackIndex = 0
+  dataHashCode = 1871012467
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 10667
+sample:
+  trackIndex = 0
+  dataHashCode = -1317831364
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 13334
+sample:
+  trackIndex = 0
+  dataHashCode = -1728189539
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 16000
+sample:
+  trackIndex = 0
+  dataHashCode = -1715881661
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 18667
+sample:
+  trackIndex = 0
+  dataHashCode = -1428554542
+  size = 1536
+  isKeyFrame = true
+  presentationTimeUs = 21334
+released = true