Add a way in the SampleConsumer to indicate that it couldn't consume

- To support looping EditedMediaItemSequences, we need a way to tell the
AssetLoader that a sample couldn't be consumed and that it should retry
later. This is necessary in case we don't know yet whether the looping
sequence should load more samples because the other sequences haven't
made sufficient progress yet.
- The decision on whether to consume a sample is based on its timestamp
so it needs to be available.

PiperOrigin-RevId: 516546026

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java

@@ -170,9 +170,10 @@ import org.checkerframework.dataflow.qual.Pure;
   }
   }
 
 
   @Override
   @Override
-  public void queueInputBuffer() {
+  public boolean queueInputBuffer() {
     DecoderInputBuffer inputBuffer = availableInputBuffers.remove();
     DecoderInputBuffer inputBuffer = availableInputBuffers.remove();
     pendingInputBuffers.add(inputBuffer);
     pendingInputBuffers.add(inputBuffer);
+    return true;
   }
   }
 
 
   @Override
   @Override

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncodedSamplePipeline.java

@@ -78,7 +78,7 @@ import java.util.concurrent.atomic.AtomicLong;
   }
   }
 
 
   @Override
   @Override
-  public void queueInputBuffer() {
+  public boolean queueInputBuffer() {
     DecoderInputBuffer inputBuffer = availableInputBuffers.remove();
     DecoderInputBuffer inputBuffer = availableInputBuffers.remove();
     if (inputBuffer.isEndOfStream()) {
     if (inputBuffer.isEndOfStream()) {
       inputEnded = true;
       inputEnded = true;
@@ -86,6 +86,7 @@ import java.util.concurrent.atomic.AtomicLong;
       inputBuffer.timeUs += mediaItemOffsetUs;
       inputBuffer.timeUs += mediaItemOffsetUs;
       pendingInputBuffers.add(inputBuffer);
       pendingInputBuffers.add(inputBuffer);
     }
     }
+    return true;
   }
   }
 
 
   @Override
   @Override

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java

@@ -63,26 +63,28 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
       return false;
     }
     }
 
 
-    if (decoder.isEnded()) {
-      checkNotNull(sampleConsumerInputBuffer.data).limit(0);
-      sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
-      sampleConsumer.queueInputBuffer();
-      isEnded = true;
-      return false;
-    }
+    ByteBuffer sampleConsumerInputData = checkNotNull(sampleConsumerInputBuffer.data);
+    if (sampleConsumerInputData.position() == 0) {
+      if (decoder.isEnded()) {
+        sampleConsumerInputData.limit(0);
+        sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
+        isEnded = sampleConsumer.queueInputBuffer();
+        return false;
+      }
 
 
-    ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
-    if (decoderOutputBuffer == null) {
-      return false;
+      ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
+      if (decoderOutputBuffer == null) {
+        return false;
+      }
+
+      sampleConsumerInputBuffer.ensureSpaceForWrite(decoderOutputBuffer.limit());
+      sampleConsumerInputBuffer.data.put(decoderOutputBuffer).flip();
+      MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
+      sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs;
+      sampleConsumerInputBuffer.setFlags(bufferInfo.flags);
+      decoder.releaseOutputBuffer(/* render= */ false);
     }
     }
 
 
-    sampleConsumerInputBuffer.ensureSpaceForWrite(decoderOutputBuffer.limit());
-    sampleConsumerInputBuffer.data.put(decoderOutputBuffer).flip();
-    MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
-    sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs;
-    sampleConsumerInputBuffer.setFlags(bufferInfo.flags);
-    decoder.releaseOutputBuffer(/* render= */ false);
-    sampleConsumer.queueInputBuffer();
-    return true;
+    return sampleConsumer.queueInputBuffer();
   }
   }
 }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderBaseRenderer.java

@@ -309,16 +309,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
       return false;
     }
     }
 
 
-    if (!readInput(sampleConsumerInputBuffer)) {
-      return false;
+    if (checkNotNull(sampleConsumerInputBuffer.data).position() == 0
+        && !sampleConsumerInputBuffer.isEndOfStream()) {
+      if (!readInput(sampleConsumerInputBuffer)) {
+        return false;
+      }
+      if (shouldDropInputBuffer(sampleConsumerInputBuffer)) {
+        return true;
+      }
     }
     }
 
 
-    if (shouldDropInputBuffer(sampleConsumerInputBuffer)) {
-      return true;
+    boolean isInputEnded = sampleConsumerInputBuffer.isEndOfStream();
+    if (!sampleConsumer.queueInputBuffer()) {
+      return false;
     }
     }
 
 
-    isEnded = sampleConsumerInputBuffer.isEndOfStream();
-    sampleConsumer.queueInputBuffer();
+    isEnded = isInputEnded;
     return !isEnded;
     return !isEnded;
   }
   }
 
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java

@@ -142,7 +142,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       return false;
       return false;
     }
     }
 
 
-    sampleConsumer.registerVideoFrame();
+    if (!sampleConsumer.registerVideoFrame(decoderOutputBufferInfo.presentationTimeUs)) {
+      return false;
+    }
+
     decoder.releaseOutputBuffer(/* render= */ true);
     decoder.releaseOutputBuffer(/* render= */ true);
     return true;
     return true;
   }
   }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ImageAssetLoader.java

@@ -72,6 +72,7 @@ public final class ImageAssetLoader implements AssetLoader {
   private final Listener listener;
   private final Listener listener;
   private final ScheduledExecutorService scheduledExecutorService;
   private final ScheduledExecutorService scheduledExecutorService;
 
 
+  @Nullable private SampleConsumer sampleConsumer;
   private @Transformer.ProgressState int progressState;
   private @Transformer.ProgressState int progressState;
 
 
   private volatile int progress;
   private volatile int progress;
@@ -87,6 +88,8 @@ public final class ImageAssetLoader implements AssetLoader {
   }
   }
 
 
   @Override
   @Override
+  // Ignore Future returned by scheduledExecutorService because failures are already handled in the
+  // runnable.
   @SuppressWarnings("FutureReturnValueIgnored")
   @SuppressWarnings("FutureReturnValueIgnored")
   public void start() {
   public void start() {
     progressState = PROGRESS_STATE_AVAILABLE;
     progressState = PROGRESS_STATE_AVAILABLE;
@@ -149,19 +152,23 @@ public final class ImageAssetLoader implements AssetLoader {
     scheduledExecutorService.shutdownNow();
     scheduledExecutorService.shutdownNow();
   }
   }
 
 
+  // Ignore Future returned by scheduledExecutorService because failures are already handled in the
+  // runnable.
   @SuppressWarnings("FutureReturnValueIgnored")
   @SuppressWarnings("FutureReturnValueIgnored")
   private void queueBitmapInternal(Bitmap bitmap, Format format) {
   private void queueBitmapInternal(Bitmap bitmap, Format format) {
     try {
     try {
-      @Nullable SampleConsumer sampleConsumer = listener.onOutputFormat(format);
       if (sampleConsumer == null) {
       if (sampleConsumer == null) {
+        sampleConsumer = listener.onOutputFormat(format);
+      }
+      // TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change
+      //    callback rather than setting duration here.
+      if (sampleConsumer == null
+          || !sampleConsumer.queueInputBitmap(
+              bitmap, editedMediaItem.durationUs, editedMediaItem.frameRate)) {
         scheduledExecutorService.schedule(
         scheduledExecutorService.schedule(
             () -> queueBitmapInternal(bitmap, format), QUEUE_BITMAP_INTERVAL_MS, MILLISECONDS);
             () -> queueBitmapInternal(bitmap, format), QUEUE_BITMAP_INTERVAL_MS, MILLISECONDS);
         return;
         return;
       }
       }
-      // TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change
-      //    callback rather than setting duration here.
-      sampleConsumer.queueInputBitmap(
-          bitmap, editedMediaItem.durationUs, editedMediaItem.frameRate);
       sampleConsumer.signalEndOfVideoInput();
       sampleConsumer.signalEndOfVideoInput();
       progress = 100;
       progress = 100;
     } catch (ExportException e) {
     } catch (ExportException e) {

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SampleConsumer.java

@@ -25,11 +25,18 @@ import com.google.android.exoplayer2.video.ColorInfo;
 public interface SampleConsumer {
 public interface SampleConsumer {
 
 
   /**
   /**
-   * Returns a buffer if the consumer is ready to accept input, and {@code null} otherwise.
+   * Returns a {@link DecoderInputBuffer}, if available.
    *
    *
-   * <p>If the consumer is ready to accept input and this method is called multiple times before
-   * {@linkplain #queueInputBuffer() queuing} input, the same {@link DecoderInputBuffer} instance is
-   * returned.
+   * <p>This {@linkplain DecoderInputBuffer buffer} should be filled with new input data and
+   * {@linkplain #queueInputBuffer() queued} to the consumer.
+   *
+   * <p>If this method returns a non-null buffer:
+   *
+   * <ul>
+   *   <li>The buffer's {@linkplain DecoderInputBuffer#data data} is non-null.
+   *   <li>The same buffer instance is returned if this method is called multiple times before
+   *       {@linkplain #queueInputBuffer() queuing} input.
+   * </ul>
    *
    *
    * <p>Should only be used for compressed data and raw audio data.
    * <p>Should only be used for compressed data and raw audio data.
    */
    */
@@ -39,30 +46,35 @@ public interface SampleConsumer {
   }
   }
 
 
   /**
   /**
-   * Informs the consumer that its input buffer contains new input.
+   * Attempts to queue new input to the consumer.
    *
    *
-   * <p>Should be called after filling the input buffer from {@link #getInputBuffer()} with new
-   * input.
+   * <p>The input buffer from {@link #getInputBuffer()} should be filled with the new input before
+   * calling this method.
    *
    *
-   * <p>An input buffer should not be used anymore after it has been queued.
+   * <p>An input buffer should not be used anymore after it has been successfully queued.
    *
    *
    * <p>Should only be used for compressed data and raw audio data.
    * <p>Should only be used for compressed data and raw audio data.
+   *
+   * @return Whether the input was successfully queued. If {@code false}, the caller should try
+   *     again later.
    */
    */
-  default void queueInputBuffer() {
+  default boolean queueInputBuffer() {
     throw new UnsupportedOperationException();
     throw new UnsupportedOperationException();
   }
   }
 
 
   /**
   /**
-   * Provides an input {@link Bitmap} to the consumer.
+   * Attempts to provide an input {@link Bitmap} to the consumer.
    *
    *
    * <p>Should only be used for image data.
    * <p>Should only be used for image data.
    *
    *
-   * @param inputBitmap The {@link Bitmap} queued to the consumer.
+   * @param inputBitmap The {@link Bitmap} to queue to the consumer.
    * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
    * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
    * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
    * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
    *     second.
    *     second.
+   * @return Whether the {@link Bitmap} was successfully queued. If {@code false}, the caller should
+   *     try again later.
    */
    */
-  default void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
+  default boolean queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
     throw new UnsupportedOperationException();
     throw new UnsupportedOperationException();
   }
   }
 
 
@@ -88,8 +100,8 @@ public interface SampleConsumer {
 
 
   /**
   /**
    * Returns the number of input video frames pending in the consumer. Pending input frames are
    * Returns the number of input video frames pending in the consumer. Pending input frames are
-   * frames that have been {@linkplain #registerVideoFrame() registered} but not processed off the
-   * {@linkplain #getInputSurface() input surface} yet.
+   * frames that have been {@linkplain #registerVideoFrame(long) registered} but not processed off
+   * the {@linkplain #getInputSurface() input surface} yet.
    *
    *
    * <p>Should only be used for raw video data.
    * <p>Should only be used for raw video data.
    */
    */
@@ -98,14 +110,18 @@ public interface SampleConsumer {
   }
   }
 
 
   /**
   /**
-   * Informs the consumer that a frame will be queued to the {@linkplain #getInputSurface() input
-   * surface}.
+   * Attempts to register a video frame to the consumer.
    *
    *
-   * <p>Must be called before rendering a frame to the input surface.
+   * <p>Each frame to consume should be registered using this method. After a frame is successfully
+   * registered, it should be rendered to the {@linkplain #getInputSurface() input surface}.
    *
    *
    * <p>Should only be used for raw video data.
    * <p>Should only be used for raw video data.
+   *
+   * @param presentationTimeUs The presentation time of the frame to register, in microseconds.
+   * @return Whether the frame was successfully registered. If {@code false}, the caller should try
+   *     again later.
    */
    */
-  default void registerVideoFrame() {
+  default boolean registerVideoFrame(long presentationTimeUs) {
     throw new UnsupportedOperationException();
     throw new UnsupportedOperationException();
   }
   }
 
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SequenceAssetLoader.java

@@ -329,34 +329,31 @@ import java.util.concurrent.atomic.AtomicInteger;
     @Nullable
     @Nullable
     @Override
     @Override
     public DecoderInputBuffer getInputBuffer() {
     public DecoderInputBuffer getInputBuffer() {
-      DecoderInputBuffer inputBuffer = sampleConsumer.getInputBuffer();
-      if (inputBuffer != null && inputBuffer.isEndOfStream()) {
-        inputBuffer.clear();
-        inputBuffer.timeUs = 0;
-      }
-      return inputBuffer;
+      return sampleConsumer.getInputBuffer();
     }
     }
 
 
     @Override
     @Override
-    public void queueInputBuffer() {
+    public boolean queueInputBuffer() {
       DecoderInputBuffer inputBuffer = checkStateNotNull(sampleConsumer.getInputBuffer());
       DecoderInputBuffer inputBuffer = checkStateNotNull(sampleConsumer.getInputBuffer());
       if (inputBuffer.isEndOfStream()) {
       if (inputBuffer.isEndOfStream()) {
         nonEndedTracks.decrementAndGet();
         nonEndedTracks.decrementAndGet();
         if (currentMediaItemIndex.get() < editedMediaItems.size() - 1) {
         if (currentMediaItemIndex.get() < editedMediaItems.size() - 1) {
+          inputBuffer.clear();
+          inputBuffer.timeUs = 0;
           if (nonEndedTracks.get() == 0) {
           if (nonEndedTracks.get() == 0) {
             switchAssetLoader();
             switchAssetLoader();
           }
           }
-          return;
+          return true;
         }
         }
       }
       }
-      sampleConsumer.queueInputBuffer();
+      return sampleConsumer.queueInputBuffer();
     }
     }
 
 
     // TODO(b/262693274): Test that concatenate 2 images or an image and a video works as expected
     // TODO(b/262693274): Test that concatenate 2 images or an image and a video works as expected
     //  once ImageAssetLoader implementation is complete.
     //  once ImageAssetLoader implementation is complete.
     @Override
     @Override
-    public void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
-      sampleConsumer.queueInputBitmap(inputBitmap, durationUs, frameRate);
+    public boolean queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
+      return sampleConsumer.queueInputBitmap(inputBitmap, durationUs, frameRate);
     }
     }
 
 
     @Override
     @Override
@@ -375,8 +372,8 @@ import java.util.concurrent.atomic.AtomicInteger;
     }
     }
 
 
     @Override
     @Override
-    public void registerVideoFrame() {
-      sampleConsumer.registerVideoFrame();
+    public boolean registerVideoFrame(long presentationTimeUs) {
+      return sampleConsumer.registerVideoFrame(presentationTimeUs);
     }
     }
 
 
     @Override
     @Override

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java

@@ -206,8 +206,9 @@ import org.checkerframework.dataflow.qual.Pure;
   }
   }
 
 
   @Override
   @Override
-  public void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
+  public boolean queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
     videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
     videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
+    return true;
   }
   }
 
 
   @Override
   @Override
@@ -221,8 +222,9 @@ import org.checkerframework.dataflow.qual.Pure;
   }
   }
 
 
   @Override
   @Override
-  public void registerVideoFrame() {
+  public boolean registerVideoFrame(long presentationTimeUs) {
     videoFrameProcessor.registerInputFrame();
     videoFrameProcessor.registerInputFrame();
+    return true;
   }
   }
 
 
   @Override
   @Override

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderTest.java

@@ -165,6 +165,8 @@ public class ExoPlayerAssetLoaderTest {
     }
     }
 
 
     @Override
     @Override
-    public void queueInputBuffer() {}
+    public boolean queueInputBuffer() {
+      return true;
+    }
   }
   }
 }
 }

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ImageAssetLoaderTest.java

@@ -133,7 +133,9 @@ public class ImageAssetLoaderTest {
   private static final class FakeSampleConsumer implements SampleConsumer {
   private static final class FakeSampleConsumer implements SampleConsumer {
 
 
     @Override
     @Override
-    public void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {}
+    public boolean queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
+      return true;
+    }
 
 
     @Override
     @Override
     public void signalEndOfVideoInput() {}
     public void signalEndOfVideoInput() {}