Skip to content

SDK / exoplayer

Go to a project
Commit e8596428 by olly Committed by kim-vde
Options

Pass correct formats to AudioSink 

The renderers are currently constructing formats that consist of their
input format with added PCM encoding. Such formats are not self-consistent,
and this only works because DefaultAudioSink ignores the rest of the
format if the format has a PCM encoding. It would not work if the sink
implementation checked the MIME type, for example, which wouldn't be a
strange or incorrect thing for it to do.

The more correct approach is to construct a new format that properly
represents the PCM that will be provided to the sink.

This change also renames supportsOutput to supportsFormat, because
AudioSink itself has both an input and an output side, and this method
is actually evaluating support on the input side of the sink.

PiperOrigin-RevId: 320396089
parent 02f8cdf1
Inline Side-by-side
Showing with 79 additions and 52 deletions
extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java
...@@ -28,6 +28,7 @@ import com.google.android.exoplayer2.drm.ExoMediaCrypto; ...@@ -28,6 +28,7 @@ import com.google.android.exoplayer2.drm.ExoMediaCrypto;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Decodes and renders audio using FFmpeg. */ /** Decodes and renders audio using FFmpeg. */
...@@ -143,13 +144,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -143,13 +144,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
private boolean isOutputSupported(Format inputFormat) { private boolean isOutputSupported(Format inputFormat) {
return shouldUseFloatOutput(inputFormat) return shouldUseFloatOutput(inputFormat)
|| supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build()); || sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT);
} }
private boolean shouldUseFloatOutput(Format inputFormat) { private boolean shouldUseFloatOutput(Format inputFormat) {
Assertions.checkNotNull(inputFormat.sampleMimeType); Assertions.checkNotNull(inputFormat.sampleMimeType);
if (!enableFloatOutput if (!enableFloatOutput || !sinkSupportsFormat(inputFormat, C.ENCODING_PCM_FLOAT)) {
|| !supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build())) {
return false; return false;
} }
switch (inputFormat.sampleMimeType) { switch (inputFormat.sampleMimeType) {
...@@ -167,4 +167,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -167,4 +167,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
} }
} }
/**
* Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output
* from the decoder for the given input format and requested output encoding.
*/
private boolean sinkSupportsFormat(Format inputFormat, @C.PcmEncoding int pcmEncoding) {
return sinkSupportsFormat(
Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate));
}
} }
extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/LibflacAudioRenderer.java
...@@ -85,22 +85,23 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer { ...@@ -85,22 +85,23 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer {
|| !MimeTypes.AUDIO_FLAC.equalsIgnoreCase(format.sampleMimeType)) { || !MimeTypes.AUDIO_FLAC.equalsIgnoreCase(format.sampleMimeType)) {
return FORMAT_UNSUPPORTED_TYPE; return FORMAT_UNSUPPORTED_TYPE;
} }
// Compute the PCM encoding that the FLAC decoder will output. // Compute the format that the FLAC decoder will output.
@C.PcmEncoding int pcmEncoding; Format outputFormat;
if (format.initializationData.isEmpty()) { if (format.initializationData.isEmpty()) {
// The initialization data might not be set if the format was obtained from a manifest (e.g. // The initialization data might not be set if the format was obtained from a manifest (e.g.
// for DASH playbacks) rather than directly from the media. In this case we assume // for DASH playbacks) rather than directly from the media. In this case we assume
// ENCODING_PCM_16BIT. If the actual encoding is different then playback will still succeed as // ENCODING_PCM_16BIT. If the actual encoding is different then playback will still succeed as
// long as the AudioSink supports it, which will always be true when using DefaultAudioSink. // long as the AudioSink supports it, which will always be true when using DefaultAudioSink.
pcmEncoding = C.ENCODING_PCM_16BIT; outputFormat =
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate);
} else { } else {
int streamMetadataOffset = int streamMetadataOffset =
FlacConstants.STREAM_MARKER_SIZE + FlacConstants.METADATA_BLOCK_HEADER_SIZE; FlacConstants.STREAM_MARKER_SIZE + FlacConstants.METADATA_BLOCK_HEADER_SIZE;
FlacStreamMetadata streamMetadata = FlacStreamMetadata streamMetadata =
new FlacStreamMetadata(format.initializationData.get(0), streamMetadataOffset); new FlacStreamMetadata(format.initializationData.get(0), streamMetadataOffset);
pcmEncoding = Util.getPcmEncoding(streamMetadata.bitsPerSample); outputFormat = getOutputFormat(streamMetadata);
} }
if (!supportsOutput(format.buildUpon().setEncoding(pcmEncoding).build())) { if (!sinkSupportsFormat(outputFormat)) {
return FORMAT_UNSUPPORTED_SUBTYPE; return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (format.drmInitData != null && format.exoMediaCryptoType == null) { } else if (format.drmInitData != null && format.exoMediaCryptoType == null) {
return FORMAT_UNSUPPORTED_DRM; return FORMAT_UNSUPPORTED_DRM;
...@@ -122,12 +123,13 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer { ...@@ -122,12 +123,13 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer {
@Override @Override
protected Format getOutputFormat() { protected Format getOutputFormat() {
Assertions.checkNotNull(streamMetadata); return getOutputFormat(Assertions.checkNotNull(streamMetadata));
return new Format.Builder() }
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(streamMetadata.channels) private static Format getOutputFormat(FlacStreamMetadata streamMetadata) {
.setSampleRate(streamMetadata.sampleRate) return Util.getPcmFormat(
.setEncoding(Util.getPcmEncoding(streamMetadata.bitsPerSample)) Util.getPcmEncoding(streamMetadata.bitsPerSample),
.build(); streamMetadata.channels,
streamMetadata.sampleRate);
} }
} }
extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java
...@@ -25,6 +25,7 @@ import com.google.android.exoplayer2.audio.DecoderAudioRenderer; ...@@ -25,6 +25,7 @@ import com.google.android.exoplayer2.audio.DecoderAudioRenderer;
import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.drm.ExoMediaCrypto;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
/** Decodes and renders audio using the native Opus decoder. */ /** Decodes and renders audio using the native Opus decoder. */
public class LibopusAudioRenderer extends DecoderAudioRenderer { public class LibopusAudioRenderer extends DecoderAudioRenderer {
...@@ -69,7 +70,8 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { ...@@ -69,7 +70,8 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
if (!OpusLibrary.isAvailable() if (!OpusLibrary.isAvailable()
|| !MimeTypes.AUDIO_OPUS.equalsIgnoreCase(format.sampleMimeType)) { || !MimeTypes.AUDIO_OPUS.equalsIgnoreCase(format.sampleMimeType)) {
return FORMAT_UNSUPPORTED_TYPE; return FORMAT_UNSUPPORTED_TYPE;
} else if (!supportsOutput(format.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build())) { } else if (!sinkSupportsFormat(
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate))) {
return FORMAT_UNSUPPORTED_SUBTYPE; return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (!drmIsSupported) { } else if (!drmIsSupported) {
return FORMAT_UNSUPPORTED_DRM; return FORMAT_UNSUPPORTED_DRM;
...@@ -99,11 +101,6 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { ...@@ -99,11 +101,6 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
@Override @Override
protected Format getOutputFormat() { protected Format getOutputFormat() {
return new Format.Builder() return Util.getPcmFormat(C.ENCODING_PCM_16BIT, channelCount, sampleRate);
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(channelCount)
.setSampleRate(sampleRate)
.setEncoding(C.ENCODING_PCM_16BIT)
.build();
} }
} }
library/common/src/main/java/com/google/android/exoplayer2/util/Util.java
...@@ -1432,13 +1432,29 @@ public final class Util { ...@@ -1432,13 +1432,29 @@ public final class Util {
} }
/** /**
* Gets a PCM {@link Format} with the specified parameters.
*
* @param pcmEncoding The {@link C.PcmEncoding}.
* @param channels The number of channels, or {@link Format#NO_VALUE} if unknown.
* @param sampleRate The sample rate in Hz, or {@link Format#NO_VALUE} if unknown.
* @return The PCM format.
*/
public static Format getPcmFormat(@C.PcmEncoding int pcmEncoding, int channels, int sampleRate) {
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(channels)
.setSampleRate(sampleRate)
.setEncoding(pcmEncoding)
.build();
}
/**
* Converts a sample bit depth to a corresponding PCM encoding constant. * Converts a sample bit depth to a corresponding PCM encoding constant.
* *
* @param bitDepth The bit depth. Supported values are 8, 16, 24 and 32. * @param bitDepth The bit depth. Supported values are 8, 16, 24 and 32.
* @return The corresponding encoding. One of {@link C#ENCODING_PCM_8BIT}, * @return The corresponding encoding. One of {@link C#ENCODING_PCM_8BIT}, {@link
* {@link C#ENCODING_PCM_16BIT}, {@link C#ENCODING_PCM_24BIT} and * C#ENCODING_PCM_16BIT}, {@link C#ENCODING_PCM_24BIT} and {@link C#ENCODING_PCM_32BIT}. If
* {@link C#ENCODING_PCM_32BIT}. If the bit depth is unsupported then * the bit depth is unsupported then {@link C#ENCODING_INVALID} is returned.
* {@link C#ENCODING_INVALID} is returned.
*/ */
@C.PcmEncoding @C.PcmEncoding
public static int getPcmEncoding(int bitDepth) { public static int getPcmEncoding(int bitDepth) {
......
library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java
...@@ -184,12 +184,12 @@ public interface AudioSink { ...@@ -184,12 +184,12 @@ public interface AudioSink {
void setListener(Listener listener); void setListener(Listener listener);
/** /**
* Returns whether the sink supports the audio format. * Returns whether the sink supports a given {@link Format}.
* *
* @param format The format of the audio. * @param format The format.
* @return Whether the sink supports the audio format. * @return Whether the sink supports the format.
*/ */
boolean supportsOutput(Format format); boolean supportsFormat(Format format);
/** /**
* Returns the playback position in the stream starting at zero, in microseconds, or * Returns the playback position in the stream starting at zero, in microseconds, or
......
library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java
...@@ -211,12 +211,12 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media ...@@ -211,12 +211,12 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media
protected abstract int supportsFormatInternal(Format format); protected abstract int supportsFormatInternal(Format format);
/** /**
* Returns whether the sink supports the audio format. * Returns whether the renderer's {@link AudioSink} supports a given {@link Format}.
* *
* @see AudioSink#supportsOutput(Format) * @see AudioSink#supportsFormat(Format)
*/ */
protected final boolean supportsOutput(Format format) { protected final boolean sinkSupportsFormat(Format format) {
return audioSink.supportsOutput(format); return audioSink.supportsFormat(format);
} }
@Override @Override
......
library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java
...@@ -419,7 +419,7 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -419,7 +419,7 @@ public final class DefaultAudioSink implements AudioSink {
} }
@Override @Override
public boolean supportsOutput(Format format) { public boolean supportsFormat(Format format) {
if (format.encoding == C.ENCODING_INVALID) { if (format.encoding == C.ENCODING_INVALID) {
return false; return false;
} }
...@@ -473,7 +473,7 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -473,7 +473,7 @@ public final class DefaultAudioSink implements AudioSink {
boolean useFloatOutput = boolean useFloatOutput =
enableFloatOutput enableFloatOutput
&& Util.isEncodingHighResolutionPcm(inputFormat.encoding) && Util.isEncodingHighResolutionPcm(inputFormat.encoding)
&& supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build()); && supportsFormat(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build());
AudioProcessor[] availableAudioProcessors = AudioProcessor[] availableAudioProcessors =
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) { if (processingEnabled) {
......
library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java
...@@ -35,8 +35,8 @@ public class ForwardingAudioSink implements AudioSink { ...@@ -35,8 +35,8 @@ public class ForwardingAudioSink implements AudioSink {
} }
@Override @Override
public boolean supportsOutput(Format format) { public boolean supportsFormat(Format format) {
return sink.supportsOutput(format); return sink.supportsFormat(format);
} }
@Override @Override
......
library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
...@@ -220,17 +220,21 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media ...@@ -220,17 +220,21 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
int tunnelingSupport = Util.SDK_INT >= 21 ? TUNNELING_SUPPORTED : TUNNELING_NOT_SUPPORTED; int tunnelingSupport = Util.SDK_INT >= 21 ? TUNNELING_SUPPORTED : TUNNELING_NOT_SUPPORTED;
boolean formatHasDrm = format.drmInitData != null || format.exoMediaCryptoType != null; boolean formatHasDrm = format.drmInitData != null || format.exoMediaCryptoType != null;
boolean supportsFormatDrm = supportsFormatDrm(format); boolean supportsFormatDrm = supportsFormatDrm(format);
// In passthrough mode, if a DRM is present we need to use a passthrough decoder to // In passthrough mode, if the format needs decryption then we need to use a passthrough
// decrypt the content. For passthrough of clear content we don't need a decoder at all. // decoder. Else we don't don't need a decoder at all.
if (supportsFormatDrm if (supportsFormatDrm
&& usePassthrough(format) && usePassthrough(format)
&& (!formatHasDrm || MediaCodecUtil.getPassthroughDecoderInfo() != null)) { && (!formatHasDrm || MediaCodecUtil.getPassthroughDecoderInfo() != null)) {
return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_NOT_SEAMLESS, tunnelingSupport); return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_NOT_SEAMLESS, tunnelingSupport);
} }
if ((MimeTypes.AUDIO_RAW.equals(format.sampleMimeType) && !audioSink.supportsOutput(format)) // If the input is PCM then it will be passed directly to the sink. Hence the sink must support
|| !audioSink.supportsOutput( // the input format directly.
format.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build())) { if (MimeTypes.AUDIO_RAW.equals(format.sampleMimeType) && !audioSink.supportsFormat(format)) {
// Assume the decoder outputs 16-bit PCM, unless the input is raw. return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE);
}
// For all other input formats, we expect the decoder to output 16-bit PCM.
if (!audioSink.supportsFormat(
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate))) {
return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE); return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE);
} }
List<MediaCodecInfo> decoderInfos = List<MediaCodecInfo> decoderInfos =
...@@ -446,7 +450,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media ...@@ -446,7 +450,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
.setChannelCount(Format.NO_VALUE) .setChannelCount(Format.NO_VALUE)
.setEncoding(C.ENCODING_E_AC3_JOC) .setEncoding(C.ENCODING_E_AC3_JOC)
.build(); .build();
if (audioSink.supportsOutput(eAc3JocFormat)) { if (audioSink.supportsFormat(eAc3JocFormat)) {
return C.ENCODING_E_AC3_JOC; return C.ENCODING_E_AC3_JOC;
} }
// E-AC3 receivers can decode JOC streams, but in 2-D rather than 3-D, so try to fall back. // E-AC3 receivers can decode JOC streams, but in 2-D rather than 3-D, so try to fall back.
...@@ -455,7 +459,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media ...@@ -455,7 +459,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
@C.Encoding int encoding = MimeTypes.getEncoding(mimeType, format.codecs); @C.Encoding int encoding = MimeTypes.getEncoding(mimeType, format.codecs);
Format passthroughFormat = format.buildUpon().setEncoding(encoding).build(); Format passthroughFormat = format.buildUpon().setEncoding(encoding).build();
if (audioSink.supportsOutput(passthroughFormat)) { if (audioSink.supportsFormat(passthroughFormat)) {
return encoding; return encoding;
} else { } else {
return C.ENCODING_INVALID; return C.ENCODING_INVALID;
......
library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java
...@@ -206,25 +206,25 @@ public final class DefaultAudioSinkTest { ...@@ -206,25 +206,25 @@ public final class DefaultAudioSinkTest {
@Config(minSdk = OLDEST_SDK, maxSdk = 20) @Config(minSdk = OLDEST_SDK, maxSdk = 20)
@Test @Test
public void doesNotSupportFloatOutputBeforeApi21() { public void doesNotSupportFloatPcmBeforeApi21() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsOutput(floatFormat)).isFalse(); assertThat(defaultAudioSink.supportsFormat(floatFormat)).isFalse();
} }
@Config(minSdk = 21, maxSdk = TARGET_SDK) @Config(minSdk = 21, maxSdk = TARGET_SDK)
@Test @Test
public void supportsFloatOutputFromApi21() { public void supportsFloatPcmFromApi21() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsOutput(floatFormat)).isTrue(); assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue();
} }
@Test @Test
public void audioSinkWithAacAudioCapabilitiesWithoutOffload_doesNotSupportAacOutput() { public void audioSinkWithAacAudioCapabilitiesWithoutOffload_doesNotSupportAac() {
DefaultAudioSink defaultAudioSink = DefaultAudioSink defaultAudioSink =
new DefaultAudioSink( new DefaultAudioSink(
new AudioCapabilities(new int[] {C.ENCODING_AAC_LC}, 2), new AudioProcessor[0]); new AudioCapabilities(new int[] {C.ENCODING_AAC_LC}, 2), new AudioProcessor[0]);
Format aacLcFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_AAC_LC).build(); Format aacLcFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_AAC_LC).build();
assertThat(defaultAudioSink.supportsOutput(aacLcFormat)).isFalse(); assertThat(defaultAudioSink.supportsFormat(aacLcFormat)).isFalse();
} }
private void configureDefaultAudioSink(int channelCount) throws AudioSink.ConfigurationException { private void configureDefaultAudioSink(int channelCount) throws AudioSink.ConfigurationException {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment