Commit 5c9c0e20 by olly Committed by Ian Baker

Improve handling of floating point audio

- DefaultAudioSink always supports floating point input. Make it
  advertise this fact.
- Remove the ability to enable/disable floating point output in
  FfmpegAudioRenderer, since this ability is now also provided on
  DefaultAudioSink.
- Let FfmpegAudioRenderer query the sink to determine whether it
  will output floating point PCM directly or resample it to 16-bit
  PCM.

PiperOrigin-RevId: 320945360
parent 21e56f57
...@@ -15,6 +15,10 @@ ...@@ -15,6 +15,10 @@
*/ */
package com.google.android.exoplayer2.ext.ffmpeg; package com.google.android.exoplayer2.ext.ffmpeg;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_UNSUPPORTED;
import android.os.Handler; import android.os.Handler;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
...@@ -22,6 +26,7 @@ import com.google.android.exoplayer2.Format; ...@@ -22,6 +26,7 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioProcessor;
import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.AudioRendererEventListener;
import com.google.android.exoplayer2.audio.AudioSink; import com.google.android.exoplayer2.audio.AudioSink;
import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport;
import com.google.android.exoplayer2.audio.DecoderAudioRenderer; import com.google.android.exoplayer2.audio.DecoderAudioRenderer;
import com.google.android.exoplayer2.audio.DefaultAudioSink; import com.google.android.exoplayer2.audio.DefaultAudioSink;
import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.drm.ExoMediaCrypto;
...@@ -41,8 +46,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -41,8 +46,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
/** The default input buffer size. */ /** The default input buffer size. */
private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6; private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6;
private final boolean enableFloatOutput;
private @MonotonicNonNull FfmpegAudioDecoder decoder; private @MonotonicNonNull FfmpegAudioDecoder decoder;
public FfmpegAudioRenderer() { public FfmpegAudioRenderer() {
...@@ -64,8 +67,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -64,8 +67,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
this( this(
eventHandler, eventHandler,
eventListener, eventListener,
new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors), new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors));
/* enableFloatOutput= */ false);
} }
/** /**
...@@ -75,21 +77,15 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -75,21 +77,15 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
* null if delivery of events is not required. * null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required. * @param eventListener A listener of events. May be null if delivery of events is not required.
* @param audioSink The sink to which audio will be output. * @param audioSink The sink to which audio will be output.
* @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the
* device/build and if the input format may have bit depth higher than 16-bit. When using
* 32-bit float output, any audio processing will be disabled, including playback speed/pitch
* adjustment.
*/ */
public FfmpegAudioRenderer( public FfmpegAudioRenderer(
@Nullable Handler eventHandler, @Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener, @Nullable AudioRendererEventListener eventListener,
AudioSink audioSink, AudioSink audioSink) {
boolean enableFloatOutput) {
super( super(
eventHandler, eventHandler,
eventListener, eventListener,
audioSink); audioSink);
this.enableFloatOutput = enableFloatOutput;
} }
@Override @Override
...@@ -103,7 +99,9 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -103,7 +99,9 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
String mimeType = Assertions.checkNotNull(format.sampleMimeType); String mimeType = Assertions.checkNotNull(format.sampleMimeType);
if (!FfmpegLibrary.isAvailable() || !MimeTypes.isAudio(mimeType)) { if (!FfmpegLibrary.isAvailable() || !MimeTypes.isAudio(mimeType)) {
return FORMAT_UNSUPPORTED_TYPE; return FORMAT_UNSUPPORTED_TYPE;
} else if (!FfmpegLibrary.supportsFormat(mimeType) || !isOutputSupported(format)) { } else if (!FfmpegLibrary.supportsFormat(mimeType)
|| (!sinkSupportsFormat(format, C.ENCODING_PCM_16BIT)
&& !sinkSupportsFormat(format, C.ENCODING_PCM_FLOAT))) {
return FORMAT_UNSUPPORTED_SUBTYPE; return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (format.drmInitData != null && format.exoMediaCryptoType == null) { } else if (format.drmInitData != null && format.exoMediaCryptoType == null) {
return FORMAT_UNSUPPORTED_DRM; return FORMAT_UNSUPPORTED_DRM;
...@@ -126,7 +124,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -126,7 +124,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
decoder = decoder =
new FfmpegAudioDecoder( new FfmpegAudioDecoder(
format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldUseFloatOutput(format)); format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldOutputFloat(format));
TraceUtil.endSection(); TraceUtil.endSection();
return decoder; return decoder;
} }
...@@ -142,31 +140,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -142,31 +140,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
.build(); .build();
} }
private boolean isOutputSupported(Format inputFormat) {
return shouldUseFloatOutput(inputFormat)
|| sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT);
}
private boolean shouldUseFloatOutput(Format inputFormat) {
Assertions.checkNotNull(inputFormat.sampleMimeType);
if (!enableFloatOutput || !sinkSupportsFormat(inputFormat, C.ENCODING_PCM_FLOAT)) {
return false;
}
switch (inputFormat.sampleMimeType) {
case MimeTypes.AUDIO_RAW:
// For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit.
return inputFormat.encoding == C.ENCODING_PCM_24BIT
|| inputFormat.encoding == C.ENCODING_PCM_32BIT
|| inputFormat.encoding == C.ENCODING_PCM_FLOAT;
case MimeTypes.AUDIO_AC3:
// AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding.
return false;
default:
// For all other formats, assume that it's worth using 32-bit float encoding.
return true;
}
}
/** /**
* Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output * Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output
* from the decoder for the given input format and requested output encoding. * from the decoder for the given input format and requested output encoding.
...@@ -175,4 +148,28 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { ...@@ -175,4 +148,28 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
return sinkSupportsFormat( return sinkSupportsFormat(
Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate)); Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate));
} }
private boolean shouldOutputFloat(Format inputFormat) {
if (!sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT)) {
// We have no choice because the sink doesn't support 16-bit integer PCM.
return true;
}
@SinkFormatSupport
int formatSupport =
getSinkFormatSupport(
Util.getPcmFormat(
C.ENCODING_PCM_FLOAT, inputFormat.channelCount, inputFormat.sampleRate));
switch (formatSupport) {
case SINK_FORMAT_SUPPORTED_DIRECTLY:
// AC-3 is always 16-bit, so there's no point using floating point. Assume that it's worth
// using for all other formats.
return !MimeTypes.AUDIO_AC3.equals(inputFormat.sampleMimeType);
case SINK_FORMAT_UNSUPPORTED:
case SINK_FORMAT_SUPPORTED_WITH_TRANSCODING:
default:
// Always prefer 16-bit PCM if the sink does not provide direct support for floating point.
return false;
}
}
} }
...@@ -16,10 +16,14 @@ ...@@ -16,10 +16,14 @@
package com.google.android.exoplayer2.audio; package com.google.android.exoplayer2.audio;
import android.media.AudioTrack; import android.media.AudioTrack;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.PlaybackParameters; import com.google.android.exoplayer2.PlaybackParameters;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
/** /**
...@@ -172,8 +176,29 @@ public interface AudioSink { ...@@ -172,8 +176,29 @@ public interface AudioSink {
} }
/** /**
* Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set. * The level of support the sink provides for a format. One of {@link
* #SINK_FORMAT_SUPPORTED_DIRECTLY}, {@link #SINK_FORMAT_SUPPORTED_WITH_TRANSCODING} or {@link
* #SINK_FORMAT_UNSUPPORTED}.
*/ */
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
SINK_FORMAT_SUPPORTED_DIRECTLY,
SINK_FORMAT_SUPPORTED_WITH_TRANSCODING,
SINK_FORMAT_UNSUPPORTED
})
@interface SinkFormatSupport {}
/** The sink supports the format directly, without the need for internal transcoding. */
int SINK_FORMAT_SUPPORTED_DIRECTLY = 2;
/**
* The sink supports the format, but needs to transcode it internally to do so. Internal
* transcoding may result in lower quality and higher CPU load in some cases.
*/
int SINK_FORMAT_SUPPORTED_WITH_TRANSCODING = 1;
/** The sink does not support the format. */
int SINK_FORMAT_UNSUPPORTED = 0;
/** Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set. */
long CURRENT_POSITION_NOT_SET = Long.MIN_VALUE; long CURRENT_POSITION_NOT_SET = Long.MIN_VALUE;
/** /**
...@@ -192,8 +217,17 @@ public interface AudioSink { ...@@ -192,8 +217,17 @@ public interface AudioSink {
boolean supportsFormat(Format format); boolean supportsFormat(Format format);
/** /**
* Returns the playback position in the stream starting at zero, in microseconds, or * Returns the level of support that the sink provides for a given {@link Format}.
* {@link #CURRENT_POSITION_NOT_SET} if it is not yet available. *
* @param format The format.
* @return The level of support provided.
*/
@SinkFormatSupport
int getFormatSupport(Format format);
/**
* Returns the playback position in the stream starting at zero, in microseconds, or {@link
* #CURRENT_POSITION_NOT_SET} if it is not yet available.
* *
* @param sourceEnded Specify {@code true} if no more input buffers will be provided. * @param sourceEnded Specify {@code true} if no more input buffers will be provided.
* @return The playback position relative to the start of playback, in microseconds. * @return The playback position relative to the start of playback, in microseconds.
......
...@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.FormatHolder; ...@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.PlayerMessage.Target; import com.google.android.exoplayer2.PlayerMessage.Target;
import com.google.android.exoplayer2.RendererCapabilities; import com.google.android.exoplayer2.RendererCapabilities;
import com.google.android.exoplayer2.audio.AudioRendererEventListener.EventDispatcher; import com.google.android.exoplayer2.audio.AudioRendererEventListener.EventDispatcher;
import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport;
import com.google.android.exoplayer2.decoder.Decoder; import com.google.android.exoplayer2.decoder.Decoder;
import com.google.android.exoplayer2.decoder.DecoderCounters; import com.google.android.exoplayer2.decoder.DecoderCounters;
import com.google.android.exoplayer2.decoder.DecoderException; import com.google.android.exoplayer2.decoder.DecoderException;
...@@ -219,6 +220,17 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media ...@@ -219,6 +220,17 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media
return audioSink.supportsFormat(format); return audioSink.supportsFormat(format);
} }
/**
* Returns the level of support that the renderer's {@link AudioSink} provides for a given {@link
* Format}.
*
* @see AudioSink#getFormatSupport(Format) (Format)
*/
@SinkFormatSupport
protected final int getSinkFormatSupport(Format format) {
return audioSink.getFormatSupport(format);
}
@Override @Override
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
if (outputStreamEnded) { if (outputStreamEnded) {
......
...@@ -380,7 +380,7 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -380,7 +380,7 @@ public final class DefaultAudioSink implements AudioSink {
boolean enableOffload) { boolean enableOffload) {
this.audioCapabilities = audioCapabilities; this.audioCapabilities = audioCapabilities;
this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain); this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain);
this.enableFloatOutput = enableFloatOutput; this.enableFloatOutput = Util.SDK_INT >= 21 && enableFloatOutput;
this.enableOffload = Util.SDK_INT >= 29 && enableOffload; this.enableOffload = Util.SDK_INT >= 29 && enableOffload;
releasingConditionVariable = new ConditionVariable(true); releasingConditionVariable = new ConditionVariable(true);
audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener()); audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener());
...@@ -420,15 +420,23 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -420,15 +420,23 @@ public final class DefaultAudioSink implements AudioSink {
@Override @Override
public boolean supportsFormat(Format format) { public boolean supportsFormat(Format format) {
return getFormatSupport(format) != SINK_FORMAT_UNSUPPORTED;
}
@Override
@SinkFormatSupport
public int getFormatSupport(Format format) {
if (format.encoding == C.ENCODING_INVALID) { if (format.encoding == C.ENCODING_INVALID) {
return false; return SINK_FORMAT_UNSUPPORTED;
} }
if (Util.isEncodingLinearPcm(format.encoding)) { if (Util.isEncodingLinearPcm(format.encoding)) {
// AudioTrack supports 16-bit integer PCM output in all platform API versions, and float if (format.encoding == C.ENCODING_PCM_16BIT
// output from platform API version 21 only. Other integer PCM encodings are resampled by this || (enableFloatOutput && format.encoding == C.ENCODING_PCM_FLOAT)) {
// sink to 16-bit PCM. We assume that the audio framework will downsample any number of return SINK_FORMAT_SUPPORTED_DIRECTLY;
// channels to the output device's required number of channels. }
return format.encoding != C.ENCODING_PCM_FLOAT || Util.SDK_INT >= 21; // We can resample all linear PCM encodings to 16-bit integer PCM, which AudioTrack is
// guaranteed to support.
return SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
} }
if (enableOffload if (enableOffload
&& isOffloadedPlaybackSupported( && isOffloadedPlaybackSupported(
...@@ -438,9 +446,12 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -438,9 +446,12 @@ public final class DefaultAudioSink implements AudioSink {
audioAttributes, audioAttributes,
format.encoderDelay, format.encoderDelay,
format.encoderPadding)) { format.encoderPadding)) {
return true; return SINK_FORMAT_SUPPORTED_DIRECTLY;
}
if (isPassthroughPlaybackSupported(format)) {
return SINK_FORMAT_SUPPORTED_DIRECTLY;
} }
return isPassthroughPlaybackSupported(format); return SINK_FORMAT_UNSUPPORTED;
} }
@Override @Override
...@@ -471,9 +482,7 @@ public final class DefaultAudioSink implements AudioSink { ...@@ -471,9 +482,7 @@ public final class DefaultAudioSink implements AudioSink {
int channelCount = inputFormat.channelCount; int channelCount = inputFormat.channelCount;
@C.Encoding int encoding = inputFormat.encoding; @C.Encoding int encoding = inputFormat.encoding;
boolean useFloatOutput = boolean useFloatOutput =
enableFloatOutput enableFloatOutput && Util.isEncodingHighResolutionPcm(inputFormat.encoding);
&& Util.isEncodingHighResolutionPcm(inputFormat.encoding)
&& supportsFormat(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build());
AudioProcessor[] availableAudioProcessors = AudioProcessor[] availableAudioProcessors =
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) { if (processingEnabled) {
......
...@@ -40,6 +40,12 @@ public class ForwardingAudioSink implements AudioSink { ...@@ -40,6 +40,12 @@ public class ForwardingAudioSink implements AudioSink {
} }
@Override @Override
@SinkFormatSupport
public int getFormatSupport(Format format) {
return sink.getFormatSupport(format);
}
@Override
public long getCurrentPositionUs(boolean sourceEnded) { public long getCurrentPositionUs(boolean sourceEnded) {
return sink.getCurrentPositionUs(sourceEnded); return sink.getCurrentPositionUs(sourceEnded);
} }
......
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
*/ */
package com.google.android.exoplayer2.audio; package com.google.android.exoplayer2.audio;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static org.robolectric.annotation.Config.OLDEST_SDK; import static org.robolectric.annotation.Config.OLDEST_SDK;
import static org.robolectric.annotation.Config.TARGET_SDK; import static org.robolectric.annotation.Config.TARGET_SDK;
...@@ -204,16 +206,46 @@ public final class DefaultAudioSinkTest { ...@@ -204,16 +206,46 @@ public final class DefaultAudioSinkTest {
.isEqualTo(8 * C.MICROS_PER_SECOND); .isEqualTo(8 * C.MICROS_PER_SECOND);
} }
@Test
public void floatPcmNeedsTranscodingIfFloatOutputDisabled() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ false);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING);
}
@Config(minSdk = OLDEST_SDK, maxSdk = 20) @Config(minSdk = OLDEST_SDK, maxSdk = 20)
@Test @Test
public void doesNotSupportFloatPcmBeforeApi21() { public void floatPcmNeedsTranscodingIfFloatOutputEnabledBeforeApi21() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ true);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isFalse(); assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING);
} }
@Config(minSdk = 21, maxSdk = TARGET_SDK) @Config(minSdk = 21, maxSdk = TARGET_SDK)
@Test @Test
public void supportsFloatPcmFromApi21() { public void floatOutputSupportedIfFloatOutputEnabledFromApi21() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ true);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_DIRECTLY);
}
@Test
public void supportsFloatPcm() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue(); assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue();
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment