Skip to content

SDK / exoplayer

Go to a project
Commit 3d8b1c99 by hschlueter Committed by bachinger
Options

Refactor AudioSamplePipeline configuration. 

The encoder and sonic are now set up in the constructor rather
than in a configuration method called from processData(). This
is more similar to VideoSamplePipeline and reduces null checks.

PiperOrigin-RevId: 420260526
parent 68613e6a
Inline Side-by-side
Showing with 38 additions and 74 deletions
libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java
...@@ -30,9 +30,6 @@ import androidx.media3.exoplayer.audio.AudioProcessor; ...@@ -30,9 +30,6 @@ import androidx.media3.exoplayer.audio.AudioProcessor;
import androidx.media3.exoplayer.audio.AudioProcessor.AudioFormat; import androidx.media3.exoplayer.audio.AudioProcessor.AudioFormat;
import androidx.media3.exoplayer.audio.SonicAudioProcessor; import androidx.media3.exoplayer.audio.SonicAudioProcessor;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them.
...@@ -42,9 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -42,9 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "AudioSamplePipeline"; private static final String TAG = "AudioSamplePipeline";
private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
private final Format inputFormat;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final Codec.EncoderFactory encoderFactory;
private final Codec decoder; private final Codec decoder;
private final DecoderInputBuffer decoderInputBuffer; private final DecoderInputBuffer decoderInputBuffer;
...@@ -52,11 +47,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -52,11 +47,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final SonicAudioProcessor sonicAudioProcessor; private final SonicAudioProcessor sonicAudioProcessor;
private final SpeedProvider speedProvider; private final SpeedProvider speedProvider;
private final Codec encoder;
private final AudioFormat encoderInputAudioFormat;
private final DecoderInputBuffer encoderInputBuffer; private final DecoderInputBuffer encoderInputBuffer;
private final DecoderInputBuffer encoderOutputBuffer; private final DecoderInputBuffer encoderOutputBuffer;
private @MonotonicNonNull AudioFormat encoderInputAudioFormat;
private @MonotonicNonNull Codec encoder;
private long nextEncoderInputBufferTimeUs; private long nextEncoderInputBufferTimeUs;
private long encoderBufferDurationRemainder; private long encoderBufferDurationRemainder;
...@@ -70,20 +65,50 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -70,20 +65,50 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory) Codec.DecoderFactory decoderFactory)
throws TransformationException { throws TransformationException {
this.inputFormat = inputFormat;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.encoderFactory = encoderFactory;
decoderInputBuffer = decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderInputBuffer = encoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer = encoderOutputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
this.decoder = decoderFactory.createForAudioDecoding(inputFormat);
sonicAudioProcessor = new SonicAudioProcessor(); sonicAudioProcessor = new SonicAudioProcessor();
sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
speedProvider = new SegmentSpeedProvider(inputFormat); speedProvider = new SegmentSpeedProvider(inputFormat);
currentSpeed = speedProvider.getSpeed(0); currentSpeed = speedProvider.getSpeed(0);
this.decoder = decoderFactory.createForAudioDecoding(inputFormat); AudioFormat encoderInputAudioFormat =
new AudioFormat(
inputFormat.sampleRate,
inputFormat.channelCount,
// The decoder uses ENCODING_PCM_16BIT by default.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
C.ENCODING_PCM_16BIT);
if (transformationRequest.flattenForSlowMotion) {
try {
encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat);
} catch (AudioProcessor.UnhandledAudioFormatException impossible) {
throw new IllegalStateException(impossible);
}
sonicAudioProcessor.setSpeed(currentSpeed);
sonicAudioProcessor.setPitch(currentSpeed);
sonicAudioProcessor.flush();
}
encoder =
encoderFactory.createForAudioEncoding(
new Format.Builder()
.setSampleMimeType(
transformationRequest.audioMimeType == null
? inputFormat.sampleMimeType
: transformationRequest.audioMimeType)
.setSampleRate(encoderInputAudioFormat.sampleRate)
.setChannelCount(encoderInputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build());
this.encoderInputAudioFormat = encoderInputAudioFormat;
} }
@Override @Override
...@@ -98,10 +123,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -98,10 +123,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() {
if (!ensureEncoderAndAudioProcessingConfigured()) {
return false;
}
if (sonicAudioProcessor.isActive()) { if (sonicAudioProcessor.isActive()) {
return feedEncoderFromSonic() || feedSonicFromDecoder(); return feedEncoderFromSonic() || feedSonicFromDecoder();
} else { } else {
...@@ -152,7 +174,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -152,7 +174,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* Attempts to pass decoder output data to the encoder, and returns whether it may be possible to * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to
* pass more data immediately by calling this method again. * pass more data immediately by calling this method again.
*/ */
@RequiresNonNull({"encoderInputAudioFormat", "encoder"})
private boolean feedEncoderFromDecoder() { private boolean feedEncoderFromDecoder() {
if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
return false; return false;
...@@ -182,7 +203,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -182,7 +203,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* Attempts to pass audio processor output data to the encoder, and returns whether it may be * Attempts to pass audio processor output data to the encoder, and returns whether it may be
* possible to pass more data immediately by calling this method again. * possible to pass more data immediately by calling this method again.
*/ */
@RequiresNonNull({"encoderInputAudioFormat", "encoder"})
private boolean feedEncoderFromSonic() { private boolean feedEncoderFromSonic() {
if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
return false; return false;
...@@ -247,7 +267,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -247,7 +267,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* Feeds as much data as possible between the current position and limit of the specified {@link * Feeds as much data as possible between the current position and limit of the specified {@link
* ByteBuffer} to the encoder, and advances its position by the number of bytes fed. * ByteBuffer} to the encoder, and advances its position by the number of bytes fed.
*/ */
@RequiresNonNull({"encoder", "encoderInputAudioFormat"})
private void feedEncoder(ByteBuffer inputBuffer) { private void feedEncoder(ByteBuffer inputBuffer) {
ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data); ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data);
int bufferLimit = inputBuffer.limit(); int bufferLimit = inputBuffer.limit();
...@@ -264,7 +283,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -264,7 +283,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
encoder.queueInputBuffer(encoderInputBuffer); encoder.queueInputBuffer(encoderInputBuffer);
} }
@RequiresNonNull("encoder")
private void queueEndOfStreamToEncoder() { private void queueEndOfStreamToEncoder() {
checkState(checkNotNull(encoderInputBuffer.data).position() == 0); checkState(checkNotNull(encoderInputBuffer.data).position() == 0);
encoderInputBuffer.timeUs = nextEncoderInputBufferTimeUs; encoderInputBuffer.timeUs = nextEncoderInputBufferTimeUs;
...@@ -274,53 +292,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -274,53 +292,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
encoder.queueInputBuffer(encoderInputBuffer); encoder.queueInputBuffer(encoderInputBuffer);
} }
/**
* Attempts to configure the {@link #encoder} and Sonic (if applicable), if they have not been
* configured yet, and returns whether they have been configured.
*/
@EnsuresNonNullIf(
expression = {"encoder", "encoderInputAudioFormat"},
result = true)
private boolean ensureEncoderAndAudioProcessingConfigured() throws TransformationException {
if (encoder != null && encoderInputAudioFormat != null) {
return true;
}
@Nullable Format decoderOutputFormat = decoder.getOutputFormat();
if (decoderOutputFormat == null) {
return false;
}
AudioFormat outputAudioFormat =
new AudioFormat(
decoderOutputFormat.sampleRate,
decoderOutputFormat.channelCount,
decoderOutputFormat.pcmEncoding);
if (transformationRequest.flattenForSlowMotion) {
try {
outputAudioFormat = sonicAudioProcessor.configure(outputAudioFormat);
flushSonicAndSetSpeed(currentSpeed);
} catch (AudioProcessor.UnhandledAudioFormatException e) {
throw TransformationException.createForAudioProcessor(
e,
"Sonic",
outputAudioFormat,
TransformationException.ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED);
}
}
encoder =
encoderFactory.createForAudioEncoding(
new Format.Builder()
.setSampleMimeType(
transformationRequest.audioMimeType == null
? inputFormat.sampleMimeType
: transformationRequest.audioMimeType)
.setSampleRate(outputAudioFormat.sampleRate)
.setChannelCount(outputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build());
encoderInputAudioFormat = outputAudioFormat;
return true;
}
private boolean isSpeedChanging(BufferInfo bufferInfo) { private boolean isSpeedChanging(BufferInfo bufferInfo) {
if (!transformationRequest.flattenForSlowMotion) { if (!transformationRequest.flattenForSlowMotion) {
return false; return false;
......
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
...@@ -73,7 +73,6 @@ public final class TransformationException extends Exception { ...@@ -73,7 +73,6 @@ public final class TransformationException extends Exception {
ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED, ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED,
ERROR_CODE_GL_INIT_FAILED, ERROR_CODE_GL_INIT_FAILED,
ERROR_CODE_GL_PROCESSING_FAILED, ERROR_CODE_GL_PROCESSING_FAILED,
ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED,
}) })
public @interface ErrorCode {} public @interface ErrorCode {}
...@@ -156,12 +155,7 @@ public final class TransformationException extends Exception { ...@@ -156,12 +155,7 @@ public final class TransformationException extends Exception {
/** Caused by a failure while using or releasing a GL program. */ /** Caused by a failure while using or releasing a GL program. */
public static final int ERROR_CODE_GL_PROCESSING_FAILED = 5002; public static final int ERROR_CODE_GL_PROCESSING_FAILED = 5002;
// Audio editing errors (6xxx). // Muxing errors (6xxx).
/** Caused by an audio processor initialization failure. */
public static final int ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED = 6001;
// Muxing errors (7xxx).
/** /**
* Caused by an output sample MIME type inferred from the input not being supported by the muxer. * Caused by an output sample MIME type inferred from the input not being supported by the muxer.
...@@ -169,7 +163,7 @@ public final class TransformationException extends Exception { ...@@ -169,7 +163,7 @@ public final class TransformationException extends Exception {
* <p>Use {@link TransformationRequest.Builder#setAudioMimeType(String)} or {@link * <p>Use {@link TransformationRequest.Builder#setAudioMimeType(String)} or {@link
* TransformationRequest.Builder#setVideoMimeType(String)} to transcode to a supported MIME type. * TransformationRequest.Builder#setVideoMimeType(String)} to transcode to a supported MIME type.
*/ */
public static final int ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED = 7001; public static final int ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED = 6001;
private static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE = private static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE =
new ImmutableBiMap.Builder<String, @ErrorCode Integer>() new ImmutableBiMap.Builder<String, @ErrorCode Integer>()
...@@ -191,7 +185,6 @@ public final class TransformationException extends Exception { ...@@ -191,7 +185,6 @@ public final class TransformationException extends Exception {
.put("ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED", ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED) .put("ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED", ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED)
.put("ERROR_CODE_GL_INIT_FAILED", ERROR_CODE_GL_INIT_FAILED) .put("ERROR_CODE_GL_INIT_FAILED", ERROR_CODE_GL_INIT_FAILED)
.put("ERROR_CODE_GL_PROCESSING_FAILED", ERROR_CODE_GL_PROCESSING_FAILED) .put("ERROR_CODE_GL_PROCESSING_FAILED", ERROR_CODE_GL_PROCESSING_FAILED)
.put("ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED", ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED)
.put( .put(
"ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED", "ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED",
ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED) ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment