Move video decoding to AssetLoader

PiperOrigin-RevId: 499454273

Move video decoding to AssetLoader
PiperOrigin-RevId: 499454273
8a8c81d5 · kimvde · Marc Baechinger · bec46b66 · 8a8c81d5 · 8a8c81d5
Commit 8a8c81d5 authored Jan 04, 2023 by kimvde Committed by Marc Baechinger Jan 04, 2023
Showing with 334 additions and 206 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java
library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AssetLoader.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SampleConsumer.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderTest.java
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
--- a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java
@@ -119,7 +119,11 @@ public interface FrameProcessor {
  /** Indicates the frame should be dropped after {@link #releaseOutputFrame(long)} is invoked. */
  long DROP_OUTPUT_FRAME = -2;
-  /** Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from. */
+  /**
+   * Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from.
+   *
+   * <p>Can be called on any thread.
+   */
  Surface getInputSurface();
  /**
@@ -141,6 +145,8 @@ public interface FrameProcessor {
   *
   * <p>Must be called before rendering a frame to the frame processor's input surface.
   *
+   * <p>Can be called on any thread.
+   *
   * @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link
   *     #setInputFrameInfo(FrameInfo)}.
   */
@@ -149,6 +155,8 @@ public interface FrameProcessor {
  /**
   * Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
   * but not processed off the {@linkplain #getInputSurface() input surface} yet.
+   *
+   * <p>Can be called on any thread.
   */
  int getPendingInputFrameCount();
@@ -193,6 +201,8 @@ public interface FrameProcessor {
  /**
   * Informs the {@code FrameProcessor} that no further input frames should be accepted.
   *
+   * <p>Can be called on any thread.
+   *
   * @throws IllegalStateException If called more than once.
   */
  void signalEndOfInput();

--- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java
+++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java
@@ -332,14 +332,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
  private final FinalMatrixTextureProcessorWrapper finalTextureProcessorWrapper;
  private final ImmutableList<GlTextureProcessor> allTextureProcessors;
-  private @MonotonicNonNull FrameInfo nextInputFrameInfo;
-  private boolean inputStreamEnded;
  /**
   * Offset compared to original media presentation time that has been added to incoming frame
   * timestamps, in microseconds.
   */
  private long previousStreamOffsetUs;
+  private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
+  private volatile boolean inputStreamEnded;
  private GlEffectsFrameProcessor(
      EGLDisplay eglDisplay,
      EGLContext eglContext,

--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AssetLoader.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AssetLoader.java
@@ -147,12 +147,11 @@ public interface AssetLoader {
     *     streamOffsetUs}), in microseconds.
     * @param streamOffsetUs The offset that will be added to the timestamps to make sure they are
     *     non-negative, in microseconds.
-     * @return The {@link SamplePipeline.Input} describing the type of sample data expected, and to
+     * @return The {@link SampleConsumer} describing the type of sample data expected, and to which
-     *     which to pass this data.
+     *     to pass this data.
-     * @throws TransformationException If an error occurs configuring the {@link
+     * @throws TransformationException If an error occurs configuring the {@link SampleConsumer}.
-     *     SamplePipeline.Input}.
     */
-    SamplePipeline.Input onTrackAdded(
+    SampleConsumer onTrackAdded(
        Format format,
        @SupportedOutputTypes int supportedOutputTypes,
        long streamStartPositionUs,

--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java
@@ -35,7 +35,7 @@ import java.util.List;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.dataflow.qual.Pure;
-/** Pipeline to apply audio processing to raw audio samples, encode them and mux them. */
+/** Pipeline to process, re-encode and mux raw audio samples. */
 /* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline {
  private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
@@ -138,11 +138,6 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  @Override
-  public boolean expectsDecodedData() {
-    return true;
-  }
-  @Override
  @Nullable
  public DecoderInputBuffer dequeueInputBuffer() {
    return hasPendingInputBuffer ? null : inputBuffer;

--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java
@@ -51,6 +51,11 @@ import com.google.android.exoplayer2.util.MimeTypes;
  }
  @Override
+  public boolean expectsDecodedData() {
+    return true;
+  }
+  @Override
  public boolean processData() throws TransformationException {
    return feedMuxer() || processDataUpToMuxer();
  }

--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java
--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SampleConsumer.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SampleConsumer.java
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.video.ColorInfo;
+/** Consumer of encoded media samples, raw audio or raw video frames. */
+public interface SampleConsumer {
+  /**
+   * Returns whether the consumer should be fed with decoded sample data. If false, encoded sample
+   * data should be fed.
+   *
+   * <p>Can be called on any thread.
+   */
+  boolean expectsDecodedData();
+  // Methods to pass compressed input or raw audio input.
+  /**
+   * Returns a buffer if the consumer is ready to accept input, and {@code null} otherwise.
+   *
+   * <p>If the consumer is ready to accept input and this method is called multiple times before
+   * {@linkplain #queueInputBuffer() queuing} input, the same buffer instance is returned.
+   *
+   * <p>Should only be used for compressed data and raw audio data.
+   */
+  @Nullable
+  default DecoderInputBuffer dequeueInputBuffer() {
+    throw new UnsupportedOperationException();
+  }
+  /**
+   * Informs the consumer that its input buffer contains new input.
+   *
+   * <p>Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new
+   * input.
+   *
+   * <p>Should only be used for compressed data and raw audio data.
+   */
+  default void queueInputBuffer() {
+    throw new UnsupportedOperationException();
+  }
+  // Methods to pass raw video input.
+  /**
+   * Returns the input {@link Surface}, where the consumer reads input frames from.
+   *
+   * <p>Should only be used for raw video data.
+   *
+   * <p>Can be called on any thread.
+   */
+  default Surface getInputSurface() {
+    throw new UnsupportedOperationException();
+  }
+  /**
+   * Returns the expected input {@link ColorInfo}.
+   *
+   * <p>Should only be used for raw video data.
+   *
+   * <p>Can be called on any thread.
+   */
+  default ColorInfo getExpectedColorInfo() {
+    throw new UnsupportedOperationException();
+  }
+  /**
+   * Returns the number of input video frames pending in the consumer. Pending input frames are
+   * frames that have been {@linkplain #registerVideoFrame() registered} but not processed off the
+   * {@linkplain #getInputSurface() input surface} yet.
+   *
+   * <p>Should only be used for raw video data.
+   *
+   * <p>Can be called on any thread.
+   */
+  default int getPendingVideoFrameCount() {
+    throw new UnsupportedOperationException();
+  }
+  /**
+   * Informs the consumer that a frame will be queued to the {@linkplain #getInputSurface() input
+   * surface}.
+   *
+   * <p>Must be called before rendering a frame to the input surface.
+   *
+   * <p>Should only be used for raw video data.
+   *
+   * <p>Can be called on any thread.
+   */
+  default void registerVideoFrame() {
+    throw new UnsupportedOperationException();
+  }
+  /**
+   * Informs the consumer that no further input frames will be rendered.
+   *
+   * <p>Should only be used for raw video data.
+   *
+   * <p>Can be called on any thread.
+   */
+  default void signalEndOfVideoInput() {
+    throw new UnsupportedOperationException();
+  }
+}
--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java
@@ -16,47 +16,12 @@
 package com.google.android.exoplayer2.transformer;
-import androidx.annotation.Nullable;
-import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 /**
 * Pipeline for processing media data.
 *
 * <p>This pipeline can be used to implement transformations of audio or video samples.
 */
-public interface SamplePipeline {
+/* package */ interface SamplePipeline extends SampleConsumer {
-  /** Input of a {@link SamplePipeline}. */
-  interface Input {
-    /** See {@link SamplePipeline#expectsDecodedData()}. */
-    boolean expectsDecodedData();
-    /** See {@link SamplePipeline#dequeueInputBuffer()}. */
-    @Nullable
-    DecoderInputBuffer dequeueInputBuffer();
-    /** See {@link SamplePipeline#queueInputBuffer()}. */
-    void queueInputBuffer();
-  }
-  /**
-   * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample
-   * data should be queued.
-   */
-  boolean expectsDecodedData();
-  /** Returns a buffer if the pipeline is ready to accept input, and {@code null} otherwise. */
-  @Nullable
-  DecoderInputBuffer dequeueInputBuffer() throws TransformationException;
-  /**
-   * Informs the pipeline that its input buffer contains new input.
-   *
-   * <p>Should be called after filling the input buffer from {@link #dequeueInputBuffer()} with new
-   * input.
-   */
-  void queueInputBuffer() throws TransformationException;
  /**
   * Processes the input data and returns whether it may be possible to process more data by calling

--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java
@@ -29,6 +29,7 @@ import android.os.HandlerThread;
 import android.os.Looper;
 import android.os.Message;
 import android.os.ParcelFileDescriptor;
+import android.view.Surface;
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
@@ -45,6 +46,7 @@ import com.google.android.exoplayer2.util.Effect;
 import com.google.android.exoplayer2.util.FrameProcessor;
 import com.google.android.exoplayer2.util.HandlerWrapper;
 import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
@@ -83,8 +85,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
  // Internal messages.
  private static final int MSG_START = 0;
  private static final int MSG_REGISTER_SAMPLE_PIPELINE = 1;
-  private static final int MSG_DEQUEUE_INPUT = 2;
+  private static final int MSG_DEQUEUE_BUFFER = 2;
-  private static final int MSG_QUEUE_INPUT = 3;
+  private static final int MSG_QUEUE_BUFFER = 3;
  private static final int MSG_DRAIN_PIPELINES = 4;
  private static final int MSG_END = 5;
  private static final int MSG_UPDATE_PROGRESS = 6;
@@ -230,11 +232,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
        case MSG_REGISTER_SAMPLE_PIPELINE:
          registerSamplePipelineInternal((SamplePipeline) msg.obj);
          break;
-        case MSG_DEQUEUE_INPUT:
+        case MSG_DEQUEUE_BUFFER:
-          dequeueInputInternal(/* samplePipelineIndex= */ msg.arg1);
+          dequeueBufferInternal(/* samplePipelineIndex= */ msg.arg1);
          break;
-        case MSG_QUEUE_INPUT:
+        case MSG_QUEUE_BUFFER:
-          samplePipelines.get(/* samplePipelineIndex= */ msg.arg1).queueInputBuffer();
+          samplePipelines.get(/* index= */ msg.arg1).queueInputBuffer();
          break;
        case MSG_DRAIN_PIPELINES:
          drainPipelinesInternal();
@@ -271,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    }
  }
-  private void dequeueInputInternal(int samplePipelineIndex) throws TransformationException {
+  private void dequeueBufferInternal(int samplePipelineIndex) throws TransformationException {
    SamplePipeline samplePipeline = samplePipelines.get(samplePipelineIndex);
    // The sample pipeline is drained before dequeuing input to maximise the chances of having an
    // input buffer to dequeue.
@@ -418,7 +420,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    }
    @Override
-    public SamplePipeline.Input onTrackAdded(
+    public SampleConsumer onTrackAdded(
        Format format,
        @AssetLoader.SupportedOutputTypes int supportedOutputTypes,
        long streamStartPositionUs,
@@ -434,7 +436,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
      SamplePipeline samplePipeline =
          getSamplePipeline(format, supportedOutputTypes, streamStartPositionUs, streamOffsetUs);
      internalHandler.obtainMessage(MSG_REGISTER_SAMPLE_PIPELINE, samplePipeline).sendToTarget();
      int samplePipelineIndex = tracksAddedCount;
      tracksAddedCount++;
@@ -458,7 +459,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
        tracksAddedCount++;
      }
-      return new SamplePipelineInput(samplePipelineIndex, samplePipeline.expectsDecodedData());
+      return new SampleConsumerImpl(samplePipelineIndex, samplePipeline);
    }
    // MuxerWrapper.Listener implementation.
@@ -523,7 +524,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
            transformationRequest,
            videoEffects,
            frameProcessorFactory,
-            decoderFactory,
            encoderFactory,
            muxerWrapper,
            /* errorConsumer= */ this::onTransformationError,
@@ -622,19 +622,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
      return false;
    }
-    private class SamplePipelineInput implements SamplePipeline.Input {
+    private class SampleConsumerImpl implements SampleConsumer {
      private final int samplePipelineIndex;
-      private final boolean expectsDecodedData;
+      private final SamplePipeline samplePipeline;
-      public SamplePipelineInput(int samplePipelineIndex, boolean expectsDecodedData) {
+      public SampleConsumerImpl(int samplePipelineIndex, SamplePipeline samplePipeline) {
        this.samplePipelineIndex = samplePipelineIndex;
-        this.expectsDecodedData = expectsDecodedData;
+        this.samplePipeline = samplePipeline;
      }
      @Override
      public boolean expectsDecodedData() {
-        return expectsDecodedData;
+        return samplePipeline.expectsDecodedData();
      }
      @Nullable
@@ -649,7 +649,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
        //  start of the sample pipelines). Having 2 thread hops per sample (one for dequeuing and
        //  one for queuing) makes transmuxing slower than it used to be.
        internalHandler
-            .obtainMessage(MSG_DEQUEUE_INPUT, samplePipelineIndex, /* unused */ 0)
+            .obtainMessage(MSG_DEQUEUE_BUFFER, samplePipelineIndex, /* unused */ 0)
            .sendToTarget();
        clock.onThreadBlocked();
        dequeueBufferConditionVariable.blockUninterruptible();
@@ -660,9 +660,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
      @Override
      public void queueInputBuffer() {
        internalHandler
-            .obtainMessage(MSG_QUEUE_INPUT, samplePipelineIndex, /* unused */ 0)
+            .obtainMessage(MSG_QUEUE_BUFFER, samplePipelineIndex, /* unused */ 0)
            .sendToTarget();
      }
+      @Override
+      public Surface getInputSurface() {
+        return samplePipeline.getInputSurface();
+      }
+      @Override
+      public ColorInfo getExpectedColorInfo() {
+        return samplePipeline.getExpectedColorInfo();
+      }
+      @Override
+      public int getPendingVideoFrameCount() {
+        return samplePipeline.getPendingVideoFrameCount();
+      }
+      @Override
+      public void registerVideoFrame() {
+        samplePipeline.registerVideoFrame();
+      }
+      @Override
+      public void signalEndOfVideoInput() {
+        samplePipeline.signalEndOfVideoInput();
+      }
    }
  }
 }
--- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
+++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
@@ -49,23 +49,15 @@ import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
 import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.dataflow.qual.Pure;
-/**
+/** Pipeline to process, re-encode and mux raw video frames. */
- * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
- */
 /* package */ final class VideoTranscodingSamplePipeline extends BaseSamplePipeline {
-  private final int maxPendingFrameCount;
-  private final DecoderInputBuffer decoderInputBuffer;
-  private final Codec decoder;
-  private final ArrayList<Long> decodeOnlyPresentationTimestamps;
  private final FrameProcessor frameProcessor;
+  private final ColorInfo frameProcessorInputColor;
  private final EncoderWrapper encoderWrapper;
  private final DecoderInputBuffer encoderOutputBuffer;
@@ -84,7 +76,6 @@ import org.checkerframework.dataflow.qual.Pure;
      TransformationRequest transformationRequest,
      ImmutableList<Effect> effects,
      FrameProcessor.Factory frameProcessorFactory,
-      Codec.DecoderFactory decoderFactory,
      Codec.EncoderFactory encoderFactory,
      MuxerWrapper muxerWrapper,
      Consumer<TransformationException> errorConsumer,
@@ -131,11 +122,8 @@ import org.checkerframework.dataflow.qual.Pure;
    finalFramePresentationTimeUs = C.TIME_UNSET;
-    decoderInputBuffer =
-        new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
    encoderOutputBuffer =
        new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
-    decodeOnlyPresentationTimestamps = new ArrayList<>();
    // The decoder rotates encoded frames for display by inputFormat.rotationDegrees.
    int decodedWidth =
@@ -169,7 +157,7 @@ import org.checkerframework.dataflow.qual.Pure;
    ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor();
    // If not tone mapping using OpenGL, the decoder will output the encoderInputColor,
    // possibly by tone mapping.
-    ColorInfo frameProcessorInputColor =
+    frameProcessorInputColor =
        isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor;
    // For consistency with the Android platform, OpenGL tone mapping outputs colors with
    // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as
@@ -236,59 +224,44 @@ import org.checkerframework.dataflow.qual.Pure;
    frameProcessor.setInputFrameInfo(
        new FrameInfo(
            decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
+  }
-    boolean isDecoderToneMappingRequired =
+  @Override
-        ColorInfo.isTransferHdr(inputFormat.colorInfo)
+  public Surface getInputSurface() {
-            && !ColorInfo.isTransferHdr(frameProcessorInputColor);
+    return frameProcessor.getInputSurface();
-    decoder =
-        decoderFactory.createForVideoDecoding(
-            inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired);
-    maxPendingFrameCount = decoder.getMaxPendingFrameCount();
  }
  @Override
-  public boolean expectsDecodedData() {
+  public ColorInfo getExpectedColorInfo() {
-    return false;
+    return frameProcessorInputColor;
  }
  @Override
-  @Nullable
+  public void registerVideoFrame() {
-  public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
+    frameProcessor.registerInputFrame();
-    return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
  }
  @Override
-  public void queueInputBuffer() throws TransformationException {
+  public int getPendingVideoFrameCount() {
-    if (decoderInputBuffer.isDecodeOnly()) {
+    return frameProcessor.getPendingInputFrameCount();
-      decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs);
  }
-    decoder.queueInputBuffer(decoderInputBuffer);
+  @Override
+  public void signalEndOfVideoInput() {
+    frameProcessor.signalEndOfInput();
  }
  @Override
  public void release() {
    frameProcessor.release();
-    decoder.release();
    encoderWrapper.release();
  }
  @Override
-  protected boolean processDataUpToMuxer() throws TransformationException {
+  protected boolean processDataUpToMuxer() {
-    if (decoder.isEnded()) {
    return false;
  }
-    boolean processedData = false;
-    while (maybeProcessDecoderOutput()) {
-      processedData = true;
-    }
-    if (decoder.isEnded()) {
-      frameProcessor.signalEndOfInput();
-    }
-    // If the decoder produced output, signal that it may be possible to process data again.
-    return processedData;
-  }
  @Override
  @Nullable
  protected Format getMuxerInputFormat() throws TransformationException {
@@ -378,46 +351,6 @@ import org.checkerframework.dataflow.qual.Pure;
  }
  /**
-   * Feeds at most one decoder output frame to the next step of the pipeline.
-   *
-   * @return Whether a frame was processed.
-   * @throws TransformationException If a problem occurs while processing the frame.
-   */
-  private boolean maybeProcessDecoderOutput() throws TransformationException {
-    @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo();
-    if (decoderOutputBufferInfo == null) {
-      return false;
-    }
-    if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) {
-      decoder.releaseOutputBuffer(/* render= */ false);
-      return true;
-    }
-    if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
-        && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
-      return false;
-    }
-    frameProcessor.registerInputFrame();
-    decoder.releaseOutputBuffer(/* render= */ true);
-    return true;
-  }
-  private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
-    // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
-    // box presentationTimeUs, creating a Long object that would need to be garbage collected.
-    int size = decodeOnlyPresentationTimestamps.size();
-    for (int i = 0; i < size; i++) {
-      if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
-        decodeOnlyPresentationTimestamps.remove(i);
-        return true;
-      }
-    }
-    return false;
-  }
-  /**
   * Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}.
   *
   * <p>The encoder is created once the {@link Surface} is {@linkplain #getSurfaceInfo(int, int)

--- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderTest.java
+++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderTest.java
@@ -68,7 +68,7 @@ public class ExoPlayerAssetLoaderTest {
          }
          @Override
-          public SamplePipeline.Input onTrackAdded(
+          public SampleConsumer onTrackAdded(
              Format format,
              @AssetLoader.SupportedOutputTypes int supportedOutputTypes,
              long streamStartPositionUs,
@@ -81,7 +81,7 @@ public class ExoPlayerAssetLoaderTest {
                  new IllegalStateException("onTrackAdded() called before onTrackCount()"));
            }
            isTrackAdded.set(true);
-            return new FakeSamplePipelineInput();
+            return new FakeSampleConsumer();
          }
          @Override
@@ -130,7 +130,7 @@ public class ExoPlayerAssetLoaderTest {
        .createAssetLoader();
  }
-  private static final class FakeSamplePipelineInput implements SamplePipeline.Input {
+  private static final class FakeSampleConsumer implements SampleConsumer {
    @Override
    public boolean expectsDecodedData() {

--- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
+++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java
@@ -633,18 +633,18 @@ public final class TransformerEndToEndTest {
  @Test
  public void startTransformation_withAssetLoaderAlwaysDecoding_pipelineExpectsDecoded()
      throws Exception {
-    AtomicReference<SamplePipeline.Input> samplePipelineInputRef = new AtomicReference<>();
+    AtomicReference<SampleConsumer> sampleConsumerRef = new AtomicReference<>();
    Transformer transformer =
        createTransformerBuilder(/* enableFallback= */ false)
            .setAssetLoaderFactory(
-                new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, samplePipelineInputRef))
+                new FakeAssetLoader.Factory(SUPPORTED_OUTPUT_TYPE_DECODED, sampleConsumerRef))
            .build();
    MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
    transformer.startTransformation(mediaItem, outputPath);
-    runLooperUntil(transformer.getApplicationLooper(), () -> samplePipelineInputRef.get() != null);
+    runLooperUntil(transformer.getApplicationLooper(), () -> sampleConsumerRef.get() != null);
-    assertThat(samplePipelineInputRef.get().expectsDecodedData()).isTrue();
+    assertThat(sampleConsumerRef.get().expectsDecodedData()).isTrue();
  }
  @Test
@@ -654,7 +654,7 @@ public final class TransformerEndToEndTest {
            .setAudioProcessors(ImmutableList.of(new SonicAudioProcessor()))
            .setAssetLoaderFactory(
                new FakeAssetLoader.Factory(
-                    SUPPORTED_OUTPUT_TYPE_ENCODED, /* samplePipelineInputRef= */ null))
+                    SUPPORTED_OUTPUT_TYPE_ENCODED, /* sampleConsumerRef= */ null))
            .build();
    MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
@@ -1077,15 +1077,15 @@ public final class TransformerEndToEndTest {
    public static final class Factory implements AssetLoader.Factory {
      private final @SupportedOutputTypes int supportedOutputTypes;
-      @Nullable private final AtomicReference<SamplePipeline.Input> samplePipelineInputRef;
+      @Nullable private final AtomicReference<SampleConsumer> sampleConsumerRef;
      @Nullable private AssetLoader.Listener listener;
      public Factory(
          @SupportedOutputTypes int supportedOutputTypes,
-          @Nullable AtomicReference<SamplePipeline.Input> samplePipelineInputRef) {
+          @Nullable AtomicReference<SampleConsumer> sampleConsumerRef) {
        this.supportedOutputTypes = supportedOutputTypes;
-        this.samplePipelineInputRef = samplePipelineInputRef;
+        this.sampleConsumerRef = sampleConsumerRef;
      }
      @Override
@@ -1136,22 +1136,21 @@ public final class TransformerEndToEndTest {
      @Override
      public AssetLoader createAssetLoader() {
-        return new FakeAssetLoader(
+        return new FakeAssetLoader(checkNotNull(listener), supportedOutputTypes, sampleConsumerRef);
-            checkNotNull(listener), supportedOutputTypes, samplePipelineInputRef);
      }
    }
    private final AssetLoader.Listener listener;
    private final @SupportedOutputTypes int supportedOutputTypes;
-    @Nullable private final AtomicReference<SamplePipeline.Input> samplePipelineInputRef;
+    @Nullable private final AtomicReference<SampleConsumer> sampleConsumerRef;
    public FakeAssetLoader(
        Listener listener,
        @SupportedOutputTypes int supportedOutputTypes,
-        @Nullable AtomicReference<SamplePipeline.Input> samplePipelineInputRef) {
+        @Nullable AtomicReference<SampleConsumer> sampleConsumerRef) {
      this.listener = listener;
      this.supportedOutputTypes = supportedOutputTypes;
-      this.samplePipelineInputRef = samplePipelineInputRef;
+      this.sampleConsumerRef = sampleConsumerRef;
    }
    @Override
@@ -1165,14 +1164,14 @@ public final class TransformerEndToEndTest {
              .setChannelCount(2)
              .build();
      try {
-        SamplePipeline.Input samplePipelineInput =
+        SampleConsumer sampleConsumer =
            listener.onTrackAdded(
                format,
                supportedOutputTypes,
                /* streamStartPositionUs= */ 0,
                /* streamOffsetUs= */ 0);
-        if (samplePipelineInputRef != null) {
+        if (sampleConsumerRef != null) {
-          samplePipelineInputRef.set(samplePipelineInput);
+          sampleConsumerRef.set(sampleConsumer);
        }
      } catch (TransformationException e) {
        throw new IllegalStateException(e);