Skip to content

SDK / exoplayer

Go to a project
Commit 18280723 by claincly Committed by Rohit Singh
Options

Move audio MIME type fallback away to ATSP 

PiperOrigin-RevId: 491660842
parent e219ac21
Inline Side-by-side
Showing with 81 additions and 61 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java
...@@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList; ...@@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
...@@ -472,9 +471,8 @@ public final class AndroidTestUtil { ...@@ -472,9 +471,8 @@ public final class AndroidTestUtil {
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) public Codec createForAudioEncoding(Format format) throws TransformationException {
throws TransformationException { return encoderFactory.createForAudioEncoding(format);
return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
} }
@Override @Override
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java
...@@ -35,7 +35,6 @@ import com.google.common.base.Ascii; ...@@ -35,7 +35,6 @@ import com.google.common.base.Ascii;
import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
...@@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner { ...@@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner {
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) public Codec createForAudioEncoding(Format format) throws TransformationException {
throws TransformationException { Codec audioEncoder = encoderFactory.createForAudioEncoding(format);
Codec audioEncoder = encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
audioEncoderName = audioEncoder.getName(); audioEncoderName = audioEncoder.getName();
return audioEncoder; return audioEncoder;
} }
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
...@@ -26,7 +26,6 @@ import androidx.test.core.app.ApplicationProvider; ...@@ -26,7 +26,6 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.MediaItem;
import java.util.List;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -140,9 +139,8 @@ public class TransformerEndToEndTest { ...@@ -140,9 +139,8 @@ public class TransformerEndToEndTest {
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) public Codec createForAudioEncoding(Format format) throws TransformationException {
throws TransformationException { return encoderFactory.createForAudioEncoding(format);
return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
} }
@Override @Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java
...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; ...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
/** /**
...@@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure;
audioProcessingPipeline.flush(); audioProcessingPipeline.flush();
String requestedMimeType =
transformationRequest.audioMimeType != null
? transformationRequest.audioMimeType
: checkNotNull(inputFormat.sampleMimeType);
Format requestedOutputFormat = Format requestedOutputFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType( .setSampleMimeType(requestedMimeType)
transformationRequest.audioMimeType == null
? inputFormat.sampleMimeType
: transformationRequest.audioMimeType)
.setSampleRate(encoderInputAudioFormat.sampleRate) .setSampleRate(encoderInputAudioFormat.sampleRate)
.setChannelCount(encoderInputAudioFormat.channelCount) .setChannelCount(encoderInputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE) .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build(); .build();
ImmutableList<String> muxerSupportedMimeTypes =
muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO);
// TODO(b/259570024): investigate overhauling fallback.
@Nullable
String supportedMimeType =
selectEncoderAndMuxerSupportedMimeType(requestedMimeType, muxerSupportedMimeTypes);
if (supportedMimeType == null) {
throw createNoSupportedMimeTypeException(requestedOutputFormat);
}
encoder = encoder =
encoderFactory.createForAudioEncoding( encoderFactory.createForAudioEncoding(
requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO)); requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build());
checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType));
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest( createFallbackTransformationRequest(
transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
...@@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure;
encoder.queueInputBuffer(encoderInputBuffer); encoder.queueInputBuffer(encoderInputBuffer);
} }
@Nullable
private static String selectEncoderAndMuxerSupportedMimeType(
String requestedMimeType, List<String> muxerSupportedMimeTypes) {
if (!EncoderUtil.getSupportedEncoders(requestedMimeType).isEmpty()) {
return requestedMimeType;
} else {
// No encoder supports the requested MIME type.
for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) {
String mimeType = muxerSupportedMimeTypes.get(i);
if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
return mimeType;
}
}
}
return null;
}
@Pure @Pure
private static TransformationRequest createFallbackTransformationRequest( private static TransformationRequest createFallbackTransformationRequest(
TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java
...@@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
: null; : null;
} }
protected static TransformationException createNoSupportedMimeTypeException(
Format requestedEncoderFormat) {
return TransformationException.createForCodec(
new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
MimeTypes.isVideo(requestedEncoderFormat.sampleMimeType),
/* isDecoder= */ false,
requestedEncoderFormat,
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
@Nullable @Nullable
@Override @Override
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
...@@ -22,9 +22,7 @@ import androidx.annotation.Nullable; ...@@ -22,9 +22,7 @@ import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.MimeTypes;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
/** /**
* Provides a layer of abstraction for interacting with decoders and encoders. * Provides a layer of abstraction for interacting with decoders and encoders.
...@@ -70,19 +68,18 @@ public interface Codec { ...@@ -70,19 +68,18 @@ public interface Codec {
/** /**
* Returns a {@link Codec} for audio encoding. * Returns a {@link Codec} for audio encoding.
* *
* <p>This method must validate that the {@link Codec} is configured to produce one of the * <p>The caller should ensure the {@linkplain Format#sampleMimeType MIME type} is supported on
* {@code allowedMimeTypes}. The {@linkplain Format#sampleMimeType sample MIME type} given in * the device before calling this method.
* {@code format} is not necessarily allowed.
* *
* @param format The {@link Format} (of the output data) used to determine the underlying * @param format The {@link Format} (of the output data) used to determine the underlying
* encoder and its configuration values. * encoder and its configuration values. {@link Format#sampleMimeType}, {@link
* @param allowedMimeTypes The non-empty list of allowed output sample {@linkplain MimeTypes * Format#sampleRate}, {@link Format#channelCount} and {@link Format#bitrate} are set to
* MIME types}. * those of the desired output video format.
* @return A {@link Codec} for audio encoding. * @return A {@link Codec} for encoding audio to the requested {@link Format#sampleMimeType MIME
* type}.
* @throws TransformationException If no suitable {@link Codec} can be created. * @throws TransformationException If no suitable {@link Codec} can be created.
*/ */
Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) Codec createForAudioEncoding(Format format) throws TransformationException;
throws TransformationException;
/** /**
* Returns a {@link Codec} for video encoding. * Returns a {@link Codec} for video encoding.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java
...@@ -172,23 +172,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { ...@@ -172,23 +172,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) public DefaultCodec createForAudioEncoding(Format format) throws TransformationException {
throws TransformationException {
// TODO(b/210591626) Add encoder selection for audio. // TODO(b/210591626) Add encoder selection for audio.
checkArgument(!allowedMimeTypes.isEmpty());
checkNotNull(format.sampleMimeType); checkNotNull(format.sampleMimeType);
if (!allowedMimeTypes.contains(format.sampleMimeType)) {
if (enableFallback) {
// TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
// capabilities limitations.
format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
} else {
throw createTransformationException(format);
}
}
MediaFormat mediaFormat = MediaFormat mediaFormat =
MediaFormat.createAudioFormat( MediaFormat.createAudioFormat(
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); format.sampleMimeType, format.sampleRate, format.channelCount);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
@Nullable @Nullable
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java
...@@ -444,9 +444,7 @@ public final class EncoderUtil { ...@@ -444,9 +444,7 @@ public final class EncoderUtil {
} }
String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes(); String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes();
for (String mimeType : supportedMimeTypes) { for (String mimeType : supportedMimeTypes) {
if (MimeTypes.isVideo(mimeType)) { encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
}
} }
} }
return encoderInfosBuilder.build(); return encoderInfosBuilder.build();
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure;
selectEncoderAndMuxerSupportedMimeType( selectEncoderAndMuxerSupportedMimeType(
requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo); requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo);
if (supportedMimeType == null) { if (supportedMimeType == null) {
throw TransformationException.createForCodec( throw createNoSupportedMimeTypeException(requestedEncoderFormat);
new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
/* isVideo= */ true,
/* isDecoder= */ false,
requestedEncoderFormat,
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
} }
encoder = encoder =
......
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
...@@ -453,18 +453,25 @@ public final class TransformerEndToEndTest { ...@@ -453,18 +453,25 @@ public final class TransformerEndToEndTest {
} }
@Test @Test
public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError() public void startTransformation_withAudioMuxerFormatUnsupported_completesSuccessfully()
throws Exception { throws Exception {
Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); // Test succeeds because MIME type fallback is mandatory.
Transformer.Listener mockListener = mock(Transformer.Listener.class);
TransformationRequest originalTransformationRequest =
new TransformationRequest.Builder().build();
TransformationRequest fallbackTransformationRequest =
new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build();
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false).addListener(mockListener).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER);
transformer.startTransformation(mediaItem, outputPath); transformer.startTransformation(mediaItem, outputPath);
TransformationException exception = TransformerTestRunner.runUntilError(transformer); TransformerTestRunner.runUntilCompleted(transformer);
assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); DumpFileAsserts.assertOutput(
assertThat(exception).hasMessageThat().contains("audio"); context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback"));
assertThat(exception.errorCode) verify(mockListener)
.isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest);
} }
@Test @Test
......
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java
...@@ -32,7 +32,6 @@ import com.google.android.exoplayer2.util.ListenerSet; ...@@ -32,7 +32,6 @@ import com.google.android.exoplayer2.util.ListenerSet;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.SurfaceInfo; import com.google.android.exoplayer2.util.SurfaceInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest { ...@@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest {
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) { public Codec createForAudioEncoding(Format format) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment