Move audio decoding to AssetLoader

PiperOrigin-RevId: 491933937

Move audio decoding to AssetLoader
PiperOrigin-RevId: 491933937
eecf7cae · kimvde · Ian Baker · ff7fe222 · eecf7cae · eecf7cae
Commit eecf7cae authored Nov 30, 2022 by kimvde Committed by Ian Baker Dec 12, 2022
Showing with 187 additions and 54 deletions
libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java
libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java
libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
@@ -41,14 +41,14 @@ import org.checkerframework.dataflow.qual.Pure;
  private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
-  private final Codec decoder;
+  private final DecoderInputBuffer inputBuffer;
-  private final DecoderInputBuffer decoderInputBuffer;
  private final AudioProcessingPipeline audioProcessingPipeline;
  private final Codec encoder;
  private final AudioFormat encoderInputAudioFormat;
  private final DecoderInputBuffer encoderInputBuffer;
  private final DecoderInputBuffer encoderOutputBuffer;
+  private boolean hasPendingInputBuffer;
  private long nextEncoderInputBufferTimeUs;
  private long encoderBufferDurationRemainder;
@@ -58,7 +58,6 @@ import org.checkerframework.dataflow.qual.Pure;
      long streamOffsetUs,
      TransformationRequest transformationRequest,
      ImmutableList<AudioProcessor> audioProcessors,
-      Codec.DecoderFactory decoderFactory,
      Codec.EncoderFactory encoderFactory,
      MuxerWrapper muxerWrapper,
      Listener listener,
@@ -72,12 +71,10 @@ import org.checkerframework.dataflow.qual.Pure;
        muxerWrapper,
        listener);
-    decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
+    inputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
    encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
    encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
-    decoder = decoderFactory.createForAudioDecoding(inputFormat);
    if (transformationRequest.flattenForSlowMotion) {
      audioProcessors =
          new ImmutableList.Builder<AudioProcessor>()
@@ -140,29 +137,33 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  @Override
+  public boolean expectsDecodedData() {
+    return true;
+  }
+  @Override
  public void release() {
    audioProcessingPipeline.reset();
-    decoder.release();
    encoder.release();
  }
  @Override
  @Nullable
-  protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException {
+  protected DecoderInputBuffer dequeueInputBufferInternal() {
-    return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
+    return hasPendingInputBuffer ? null : inputBuffer;
  }
  @Override
-  protected void queueInputBufferInternal() throws TransformationException {
+  protected void queueInputBufferInternal() {
-    decoder.queueInputBuffer(decoderInputBuffer);
+    hasPendingInputBuffer = true;
  }
  @Override
  protected boolean processDataUpToMuxer() throws TransformationException {
    if (audioProcessingPipeline.isOperational()) {
-      return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromDecoder();
+      return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromInput();
    } else {
-      return feedEncoderFromDecoder();
+      return feedEncoderFromInput();
    }
  }
@@ -195,27 +196,25 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  /**
-   * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to
+   * Attempts to pass input data to the encoder.
-   * pass more data immediately by calling this method again.
+   *
+   * @return Whether it may be possible to feed more data immediately by calling this method again.
   */
-  private boolean feedEncoderFromDecoder() throws TransformationException {
+  private boolean feedEncoderFromInput() throws TransformationException {
-    if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
+    if (!hasPendingInputBuffer || !encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
      return false;
    }
-    if (decoder.isEnded()) {
+    if (inputBuffer.isEndOfStream()) {
      queueEndOfStreamToEncoder();
+      hasPendingInputBuffer = false;
      return false;
    }
-    @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
+    ByteBuffer inputData = checkNotNull(inputBuffer.data);
-    if (decoderOutputBuffer == null) {
+    feedEncoder(inputData);
-      return false;
+    if (!inputData.hasRemaining()) {
-    }
+      hasPendingInputBuffer = false;
-    feedEncoder(decoderOutputBuffer);
-    if (!decoderOutputBuffer.hasRemaining()) {
-      decoder.releaseOutputBuffer(/* render= */ false);
    }
    return true;
  }
@@ -223,7 +222,7 @@ import org.checkerframework.dataflow.qual.Pure;
  /**
   * Attempts to feed audio processor output data to the encoder.
   *
-   * @return Whether more data can be fed immediately, by calling this method again.
+   * @return Whether it may be possible to feed more data immediately by calling this method again.
   */
  private boolean feedEncoderFromProcessingPipeline() throws TransformationException {
    if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
@@ -244,28 +243,28 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  /**
-   * Attempts to feed decoder output data to the {@link AudioProcessingPipeline}.
+   * Attempts to feed input data to the {@link AudioProcessingPipeline}.
   *
   * @return Whether it may be possible to feed more data immediately by calling this method again.
   */
-  private boolean feedProcessingPipelineFromDecoder() throws TransformationException {
+  private boolean feedProcessingPipelineFromInput() {
-    if (decoder.isEnded()) {
+    if (!hasPendingInputBuffer) {
-      audioProcessingPipeline.queueEndOfStream();
      return false;
    }
-    checkState(!audioProcessingPipeline.isEnded());
-    @Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
+    if (inputBuffer.isEndOfStream()) {
-    if (decoderOutputBuffer == null) {
+      audioProcessingPipeline.queueEndOfStream();
+      hasPendingInputBuffer = false;
      return false;
    }
+    checkState(!audioProcessingPipeline.isEnded());
-    audioProcessingPipeline.queueInput(decoderOutputBuffer);
+    ByteBuffer inputData = checkNotNull(inputBuffer.data);
-    if (decoderOutputBuffer.hasRemaining()) {
+    audioProcessingPipeline.queueInput(inputData);
+    if (inputData.hasRemaining()) {
      return false;
    }
-    // Decoder output buffer was fully consumed by the processing pipeline.
+    hasPendingInputBuffer = false;
-    decoder.releaseOutputBuffer(/* render= */ false);
    return true;
  }

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java
@@ -69,6 +69,7 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
      boolean removeAudio,
      boolean removeVideo,
      MediaSource.Factory mediaSourceFactory,
+      Codec.DecoderFactory decoderFactory,
      Looper looper,
      Listener listener,
      Clock clock) {
@@ -89,7 +90,9 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
                DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
            .build();
    ExoPlayer.Builder playerBuilder =
-        new ExoPlayer.Builder(context, new RenderersFactoryImpl(removeAudio, removeVideo, listener))
+        new ExoPlayer.Builder(
+                context,
+                new RenderersFactoryImpl(removeAudio, removeVideo, decoderFactory, listener))
            .setMediaSourceFactory(mediaSourceFactory)
            .setTrackSelector(trackSelector)
            .setLoadControl(loadControl)
@@ -120,14 +123,17 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
    private final TransformerMediaClock mediaClock;
    private final boolean removeAudio;
    private final boolean removeVideo;
+    private final Codec.DecoderFactory decoderFactory;
    private final ExoPlayerAssetLoader.Listener assetLoaderListener;
    public RenderersFactoryImpl(
        boolean removeAudio,
        boolean removeVideo,
+        Codec.DecoderFactory decoderFactory,
        ExoPlayerAssetLoader.Listener assetLoaderListener) {
      this.removeAudio = removeAudio;
      this.removeVideo = removeVideo;
+      this.decoderFactory = decoderFactory;
      this.assetLoaderListener = assetLoaderListener;
      mediaClock = new TransformerMediaClock();
    }
@@ -144,12 +150,14 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
      int index = 0;
      if (!removeAudio) {
        renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener);
+            new ExoPlayerAssetLoaderRenderer(
+                C.TRACK_TYPE_AUDIO, decoderFactory, mediaClock, assetLoaderListener);
        index++;
      }
      if (!removeVideo) {
        renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener);
+            new ExoPlayerAssetLoaderRenderer(
+                C.TRACK_TYPE_VIDEO, decoderFactory, mediaClock, assetLoaderListener);
        index++;
      }
      return renderers;

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
@@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED;
 import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT;
+import android.media.MediaCodec;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
@@ -30,6 +31,7 @@ import androidx.media3.exoplayer.FormatHolder;
 import androidx.media3.exoplayer.MediaClock;
 import androidx.media3.exoplayer.RendererCapabilities;
 import androidx.media3.exoplayer.source.SampleStream.ReadDataResult;
+import java.nio.ByteBuffer;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@@ -38,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
  private static final String TAG = "ExoPlayerAssetLoaderRenderer";
+  private final Codec.DecoderFactory decoderFactory;
  private final TransformerMediaClock mediaClock;
  private final ExoPlayerAssetLoader.Listener assetLoaderListener;
  private final DecoderInputBuffer decoderInputBuffer;
@@ -45,14 +48,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
  private boolean isTransformationRunning;
  private long streamStartPositionUs;
  private long streamOffsetUs;
+  private @MonotonicNonNull Codec decoder;
+  @Nullable private ByteBuffer pendingDecoderOutputBuffer;
  private SamplePipeline.@MonotonicNonNull Input samplePipelineInput;
  private boolean isEnded;
  public ExoPlayerAssetLoaderRenderer(
      int trackType,
+      Codec.DecoderFactory decoderFactory,
      TransformerMediaClock mediaClock,
      ExoPlayerAssetLoader.Listener assetLoaderListener) {
    super(trackType);
+    this.decoderFactory = decoderFactory;
    this.mediaClock = mediaClock;
    this.assetLoaderListener = assetLoaderListener;
    decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
@@ -99,7 +106,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
        return;
      }
-      while (feedPipelineFromInput()) {}
+      if (samplePipelineInput.expectsDecodedData()) {
+        while (feedPipelineFromDecoder() || feedDecoderFromInput()) {}
+      } else {
+        while (feedPipelineFromInput()) {}
+      }
    } catch (TransformationException e) {
      isTransformationRunning = false;
      assetLoaderListener.onError(e);
@@ -128,6 +139,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    isTransformationRunning = false;
  }
+  @Override
+  protected void onReset() {
+    if (decoder != null) {
+      decoder.release();
+    }
+  }
  @EnsuresNonNullIf(expression = "samplePipelineInput", result = true)
  private boolean ensureConfigured() throws TransformationException {
    if (samplePipelineInput != null) {
@@ -143,6 +161,74 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    Format inputFormat = checkNotNull(formatHolder.format);
    samplePipelineInput =
        assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs);
+    if (samplePipelineInput.expectsDecodedData()) {
+      decoder = decoderFactory.createForAudioDecoding(inputFormat);
+    }
+    return true;
+  }
+  /**
+   * Attempts to read decoded data and pass it to the sample pipeline.
+   *
+   * @return Whether it may be possible to read more data immediately by calling this method again.
+   * @throws TransformationException If an error occurs in the decoder or in the {@link
+   *     SamplePipeline}.
+   */
+  @RequiresNonNull("samplePipelineInput")
+  private boolean feedPipelineFromDecoder() throws TransformationException {
+    @Nullable
+    DecoderInputBuffer samplePipelineInputBuffer = samplePipelineInput.dequeueInputBuffer();
+    if (samplePipelineInputBuffer == null) {
+      return false;
+    }
+    Codec decoder = checkNotNull(this.decoder);
+    if (pendingDecoderOutputBuffer != null) {
+      if (pendingDecoderOutputBuffer.hasRemaining()) {
+        return false;
+      } else {
+        decoder.releaseOutputBuffer(/* render= */ false);
+        pendingDecoderOutputBuffer = null;
+      }
+    }
+    if (decoder.isEnded()) {
+      samplePipelineInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
+      samplePipelineInput.queueInputBuffer();
+      isEnded = true;
+      return false;
+    }
+    pendingDecoderOutputBuffer = decoder.getOutputBuffer();
+    if (pendingDecoderOutputBuffer == null) {
+      return false;
+    }
+    samplePipelineInputBuffer.data = pendingDecoderOutputBuffer;
+    MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
+    samplePipelineInputBuffer.timeUs = bufferInfo.presentationTimeUs;
+    samplePipelineInputBuffer.setFlags(bufferInfo.flags);
+    samplePipelineInput.queueInputBuffer();
+    return true;
+  }
+  /**
+   * Attempts to read input data and pass it to the decoder.
+   *
+   * @return Whether it may be possible to read more data immediately by calling this method again.
+   * @throws TransformationException If an error occurs in the decoder.
+   */
+  private boolean feedDecoderFromInput() throws TransformationException {
+    Codec decoder = checkNotNull(this.decoder);
+    if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) {
+      return false;
+    }
+    if (!readInput(decoderInputBuffer)) {
+      return false;
+    }
+    decoder.queueInputBuffer(decoderInputBuffer);
    return true;
  }
@@ -159,18 +245,32 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
      return false;
    }
-    @ReadDataResult
+    if (!readInput(samplePipelineInputBuffer)) {
-    int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0);
+      return false;
+    }
+    samplePipelineInput.queueInputBuffer();
+    if (samplePipelineInputBuffer.isEndOfStream()) {
+      isEnded = true;
+      return false;
+    }
+    return true;
+  }
+  /**
+   * Attempts to populate {@code buffer} with input data.
+   *
+   * @param buffer The buffer to populate.
+   * @return Whether the {@code buffer} has been populated.
+   */
+  private boolean readInput(DecoderInputBuffer buffer) {
+    @ReadDataResult int result = readSource(getFormatHolder(), buffer, /* readFlags= */ 0);
    switch (result) {
      case C.RESULT_BUFFER_READ:
-        samplePipelineInputBuffer.flip();
+        buffer.flip();
-        if (samplePipelineInputBuffer.isEndOfStream()) {
+        if (!buffer.isEndOfStream()) {
-          samplePipelineInput.queueInputBuffer();
+          mediaClock.updateTimeForTrackType(getTrackType(), buffer.timeUs);
-          isEnded = true;
-          return false;
        }
-        mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs);
-        samplePipelineInput.queueInputBuffer();
        return true;
      case C.RESULT_FORMAT_READ:
        throw new IllegalStateException("Format changes are not supported.");

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java
@@ -49,6 +49,11 @@ import androidx.media3.decoder.DecoderInputBuffer;
  }
  @Override
+  public boolean expectsDecodedData() {
+    return false;
+  }
+  @Override
  public void release() {}
  @Override

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
@@ -29,6 +29,9 @@ import androidx.media3.decoder.DecoderInputBuffer;
  /** Input of a {@link SamplePipeline}. */
  interface Input {
+    /** See {@link SamplePipeline#expectsDecodedData()}. */
+    boolean expectsDecodedData();
    /** See {@link SamplePipeline#dequeueInputBuffer()}. */
    @Nullable
    DecoderInputBuffer dequeueInputBuffer();
@@ -56,6 +59,12 @@ import androidx.media3.decoder.DecoderInputBuffer;
    void onTransformationError(TransformationException exception);
  }
+  /**
+   * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample
+   * data should be queued.
+   */
+  boolean expectsDecodedData();
  /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */
  @Nullable
  DecoderInputBuffer dequeueInputBuffer() throws TransformationException;

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
@@ -163,6 +163,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
            removeAudio,
            removeVideo,
            mediaSourceFactory,
+            decoderFactory,
            internalLooper,
            componentListener,
            clock);
@@ -409,7 +410,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
      int samplePipelineIndex = tracksAddedCount;
      tracksAddedCount++;
-      return new SamplePipelineInput(samplePipelineIndex);
+      return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData());
    }
    @Override
@@ -458,7 +459,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
            streamOffsetUs,
            transformationRequest,
            audioProcessors,
-            decoderFactory,
            encoderFactory,
            muxerWrapper,
            /* listener= */ this,
@@ -573,9 +573,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    private class SamplePipelineInput implements SamplePipeline.Input {
      private final int samplePipelineIndex;
+      private final boolean expectsDecodedData;
-      public SamplePipelineInput(int samplePipelineIndex) {
+      public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) {
        this.samplePipelineIndex = samplePipelineIndex;
+        this.expectsDecodedData = expectsDecodedData;
+      }
+      @Override
+      public boolean expectsDecodedData() {
+        return expectsDecodedData;
      }
      @Nullable

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@@ -208,6 +208,11 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  @Override
+  public boolean expectsDecodedData() {
+    return false;
+  }
+  @Override
  public void release() {
    frameProcessor.release();
    decoder.release();