Skip to content

SDK / exoplayer

Go to a project
Commit 4cf91065 by hschlueter Committed by Ian Baker
Options

Transcode to a muxer-supported sample MIME type. 

If the output sample MIME type is inferred from the input
but is not supported by the muxer, we fallback to transcoding
to a supported sample MIME type.
The audio and video renderers need to make sure not to select the PassthroughSamplePipeline for this case. Which sample MIME type
to choose is decided by the EncoderFactory.

PiperOrigin-RevId: 423272812
parent e2cf266b
Expand all
Inline Side-by-side
Showing with 131 additions and 40 deletions
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java
...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.audio.SonicAudioProcessor; ...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.audio.SonicAudioProcessor;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
/** /**
...@@ -63,8 +64,9 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -63,8 +64,9 @@ import org.checkerframework.dataflow.qual.Pure;
public AudioSamplePipeline( public AudioSamplePipeline(
Format inputFormat, Format inputFormat,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
FallbackListener fallbackListener) FallbackListener fallbackListener)
throws TransformationException { throws TransformationException {
decoderInputBuffer = decoderInputBuffer =
...@@ -110,7 +112,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -110,7 +112,7 @@ import org.checkerframework.dataflow.qual.Pure;
.setChannelCount(encoderInputAudioFormat.channelCount) .setChannelCount(encoderInputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE) .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build(); .build();
encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat); encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes);
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackRequest( createFallbackRequest(
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
...@@ -30,6 +30,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; ...@@ -30,6 +30,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -54,7 +55,7 @@ public final class Codec { ...@@ -54,7 +55,7 @@ public final class Codec {
* @param format The {@link Format} (of the input data) used to determine the underlying {@link * @param format The {@link Format} (of the input data) used to determine the underlying {@link
* MediaCodec} and its configuration values. * MediaCodec} and its configuration values.
* @return A configured and started decoder wrapper. * @return A configured and started decoder wrapper.
* @throws TransformationException If the underlying codec cannot be created. * @throws TransformationException If no suitable codec can be created.
*/ */
Codec createForAudioDecoding(Format format) throws TransformationException; Codec createForAudioDecoding(Format format) throws TransformationException;
...@@ -65,7 +66,7 @@ public final class Codec { ...@@ -65,7 +66,7 @@ public final class Codec {
* MediaCodec} and its configuration values. * MediaCodec} and its configuration values.
* @param outputSurface The {@link Surface} to which the decoder output is rendered. * @param outputSurface The {@link Surface} to which the decoder output is rendered.
* @return A configured and started decoder wrapper. * @return A configured and started decoder wrapper.
* @throws TransformationException If the underlying codec cannot be created. * @throws TransformationException If no suitable codec can be created.
*/ */
Codec createForVideoDecoding(Format format, Surface outputSurface) Codec createForVideoDecoding(Format format, Surface outputSurface)
throws TransformationException; throws TransformationException;
...@@ -80,25 +81,39 @@ public final class Codec { ...@@ -80,25 +81,39 @@ public final class Codec {
/** /**
* Returns a {@link Codec} for audio encoding. * Returns a {@link Codec} for audio encoding.
* *
* <p>This method must validate that the {@link Codec} is configured to produce one of the
* {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
* format} is not necessarily allowed.
*
* @param format The {@link Format} (of the output data) used to determine the underlying {@link * @param format The {@link Format} (of the output data) used to determine the underlying {@link
* MediaCodec} and its configuration values. * MediaCodec} and its configuration values.
* @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
* types}.
* @return A configured and started encoder wrapper. * @return A configured and started encoder wrapper.
* @throws TransformationException If the underlying codec cannot be created. * @throws TransformationException If no suitable codec can be created.
*/ */
Codec createForAudioEncoding(Format format) throws TransformationException; Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException;
/** /**
* Returns a {@link Codec} for video encoding. * Returns a {@link Codec} for video encoding.
* *
* <p>This method must validate that the {@link Codec} is configured to produce one of the
* {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
* format} is not necessarily allowed.
*
* @param format The {@link Format} (of the output data) used to determine the underlying {@link * @param format The {@link Format} (of the output data) used to determine the underlying {@link
* MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link * MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
* Format#width} and {@link Format#height} must be set to those of the desired output video * Format#width} and {@link Format#height} must be set to those of the desired output video
* format. {@link Format#rotationDegrees} should be 0. The video should always be in * format. {@link Format#rotationDegrees} should be 0. The video should always be in
* landscape orientation. * landscape orientation.
* @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
* types}.
* @return A configured and started encoder wrapper. * @return A configured and started encoder wrapper.
* @throws TransformationException If the underlying codec cannot be created. * @throws TransformationException If no suitable codec can be created.
*/ */
Codec createForVideoEncoding(Format format) throws TransformationException; Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException;
} }
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float. // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java
...@@ -34,13 +34,15 @@ import com.google.android.exoplayer2.util.MediaFormatUtil; ...@@ -34,13 +34,15 @@ import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.TraceUtil;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */ /** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */
/* package */ final class DefaultCodecFactory /* package */ final class DefaultCodecFactory
implements Codec.DecoderFactory, Codec.EncoderFactory { implements Codec.DecoderFactory, Codec.EncoderFactory {
// TODO(b/214973843): Add option to disable fallback.
// TODO(b/210591626) Fall back adaptively to H265 if possible. // TODO(b/210591626): Fall back adaptively to H265 if possible.
private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264; private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264;
private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface; private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface;
private static final int DEFAULT_FRAME_RATE = 60; private static final int DEFAULT_FRAME_RATE = 60;
...@@ -85,7 +87,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -85,7 +87,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
@Override @Override
public Codec createForAudioEncoding(Format format) throws TransformationException { public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
checkArgument(!allowedMimeTypes.isEmpty());
if (!allowedMimeTypes.contains(format.sampleMimeType)) {
// TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
// capabilities limitations.
format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
}
MediaFormat mediaFormat = MediaFormat mediaFormat =
MediaFormat.createAudioFormat( MediaFormat.createAudioFormat(
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
...@@ -100,7 +109,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -100,7 +109,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
@Override @Override
public Codec createForVideoEncoding(Format format) throws TransformationException { public Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
checkArgument(format.width != Format.NO_VALUE); checkArgument(format.width != Format.NO_VALUE);
checkArgument(format.height != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE);
// According to interface Javadoc, format.rotationDegrees should be 0. The video should always // According to interface Javadoc, format.rotationDegrees should be 0. The video should always
...@@ -108,7 +118,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -108,7 +118,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
checkArgument(format.height <= format.width); checkArgument(format.height <= format.width);
checkArgument(format.rotationDegrees == 0); checkArgument(format.rotationDegrees == 0);
checkNotNull(format.sampleMimeType); checkNotNull(format.sampleMimeType);
format = getVideoEncoderSupportedFormat(format);
checkArgument(!allowedMimeTypes.isEmpty());
format = getVideoEncoderSupportedFormat(format, allowedMimeTypes);
MediaFormat mediaFormat = MediaFormat mediaFormat =
MediaFormat.createVideoFormat( MediaFormat.createVideoFormat(
...@@ -191,14 +204,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -191,14 +204,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
@RequiresNonNull("#1.sampleMimeType") @RequiresNonNull("#1.sampleMimeType")
private static Format getVideoEncoderSupportedFormat(Format requestedFormat) private static Format getVideoEncoderSupportedFormat(
throws TransformationException { Format requestedFormat, List<String> allowedMimeTypes) throws TransformationException {
String mimeType = requestedFormat.sampleMimeType; String mimeType = requestedFormat.sampleMimeType;
Format.Builder formatBuilder = requestedFormat.buildUpon(); Format.Builder formatBuilder = requestedFormat.buildUpon();
// TODO(b/210591626) Implement encoder filtering. // TODO(b/210591626) Implement encoder filtering.
if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { if (!allowedMimeTypes.contains(mimeType)
mimeType = DEFAULT_FALLBACK_MIME_TYPE; || EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
mimeType =
allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE)
? DEFAULT_FALLBACK_MIME_TYPE
: allowedMimeTypes.get(0);
if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
throw createTransformationException( throw createTransformationException(
new IllegalArgumentException( new IllegalArgumentException(
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java
...@@ -68,23 +68,18 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; ...@@ -68,23 +68,18 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
return false; return false;
} }
Format inputFormat = checkNotNull(formatHolder.format); Format inputFormat = checkNotNull(formatHolder.format);
String sampleMimeType = checkNotNull(inputFormat.sampleMimeType);
if (transformationRequest.audioMimeType == null
&& !muxerWrapper.supportsSampleMimeType(sampleMimeType)) {
throw TransformationException.createForMuxer(
new IllegalArgumentException(
"The output sample MIME inferred from the input format is not supported by the muxer."
+ " Sample MIME type: "
+ sampleMimeType),
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
if (shouldPassthrough(inputFormat)) { if (shouldPassthrough(inputFormat)) {
samplePipeline = samplePipeline =
new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
} else { } else {
samplePipeline = samplePipeline =
new AudioSamplePipeline( new AudioSamplePipeline(
inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener); inputFormat,
transformationRequest,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
fallbackListener);
} }
return true; return true;
} }
...@@ -94,6 +89,10 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; ...@@ -94,6 +89,10 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
&& !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) {
return false; return false;
} }
if (transformationRequest.audioMimeType == null
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
return false;
}
if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) {
return false; return false;
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java
...@@ -77,16 +77,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -77,16 +77,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return false; return false;
} }
Format inputFormat = checkNotNull(formatHolder.format); Format inputFormat = checkNotNull(formatHolder.format);
String sampleMimeType = checkNotNull(inputFormat.sampleMimeType);
if (transformationRequest.videoMimeType == null
&& !muxerWrapper.supportsSampleMimeType(sampleMimeType)) {
throw TransformationException.createForMuxer(
new IllegalArgumentException(
"The output sample MIME inferred from the input format is not supported by the muxer."
+ " Sample MIME type: "
+ sampleMimeType),
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
if (shouldPassthrough(inputFormat)) { if (shouldPassthrough(inputFormat)) {
samplePipeline = samplePipeline =
new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
...@@ -96,8 +86,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -96,8 +86,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
context, context,
inputFormat, inputFormat,
transformationRequest, transformationRequest,
encoderFactory,
decoderFactory, decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
fallbackListener, fallbackListener,
debugViewProvider); debugViewProvider);
} }
...@@ -112,6 +103,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -112,6 +103,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) {
return false; return false;
} }
if (transformationRequest.videoMimeType == null
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
return false;
}
if (transformationRequest.outputHeight != C.LENGTH_UNSET if (transformationRequest.outputHeight != C.LENGTH_UNSET
&& transformationRequest.outputHeight != inputFormat.height) { && transformationRequest.outputHeight != inputFormat.height) {
return false; return false;
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java
...@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.C; ...@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
...@@ -54,8 +55,9 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -54,8 +55,9 @@ import org.checkerframework.dataflow.qual.Pure;
Context context, Context context,
Format inputFormat, Format inputFormat,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
FallbackListener fallbackListener, FallbackListener fallbackListener,
Transformer.DebugViewProvider debugViewProvider) Transformer.DebugViewProvider debugViewProvider)
throws TransformationException { throws TransformationException {
...@@ -115,7 +117,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -115,7 +117,7 @@ import org.checkerframework.dataflow.qual.Pure;
? transformationRequest.videoMimeType ? transformationRequest.videoMimeType
: inputFormat.sampleMimeType) : inputFormat.sampleMimeType)
.build(); .build();
encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat); encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat, allowedOutputMimeTypes);
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackRequest( createFallbackRequest(
transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
......
testdata/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump 0 → 100644
containerMimeType = video/mp4
format 0:
sampleMimeType = audio/mp4a-latm
channelCount = 6
sampleRate = 48000
pcmEncoding = 2
sample:
trackIndex = 0
dataHashCode = 1896404418
size = 1536
isKeyFrame = true
presentationTimeUs = 0
sample:
trackIndex = 0
dataHashCode = -2134951116
size = 1536
isKeyFrame = true
presentationTimeUs = 2667
sample:
trackIndex = 0
dataHashCode = 97556101
size = 1536
isKeyFrame = true
presentationTimeUs = 5334
sample:
trackIndex = 0
dataHashCode = -1448980924
size = 1536
isKeyFrame = true
presentationTimeUs = 8000
sample:
trackIndex = 0
dataHashCode = 1871012467
size = 1536
isKeyFrame = true
presentationTimeUs = 10667
sample:
trackIndex = 0
dataHashCode = -1317831364
size = 1536
isKeyFrame = true
presentationTimeUs = 13334
sample:
trackIndex = 0
dataHashCode = -1728189539
size = 1536
isKeyFrame = true
presentationTimeUs = 16000
sample:
trackIndex = 0
dataHashCode = -1715881661
size = 1536
isKeyFrame = true
presentationTimeUs = 18667
sample:
trackIndex = 0
dataHashCode = -1428554542
size = 1536
isKeyFrame = true
presentationTimeUs = 21334
released = true
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment