Move progress updates to the AssetLoader

This is necessary to move video slow motion flattening to the
AssetLoader because this step can change the duration. As we use the
duration before flattening to calculate the progress, we must also use
the position before flattening.

PiperOrigin-RevId: 493291990

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java

@@ -64,7 +64,6 @@ import org.checkerframework.dataflow.qual.Pure;
       long forceSilentAudioDurationUs,
       Codec.EncoderFactory encoderFactory,
       MuxerWrapper muxerWrapper,
-      Listener listener,
       FallbackListener fallbackListener)
       throws TransformationException {
     super(
@@ -72,8 +71,7 @@ import org.checkerframework.dataflow.qual.Pure;
         streamStartPositionUs,
         streamOffsetUs,
         transformationRequest.flattenForSlowMotion,
-        muxerWrapper,
-        listener);
+        muxerWrapper);
 
     if (forceSilentAudioDurationUs != C.TIME_UNSET) {
       silentAudioGenerator =

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java

@@ -33,7 +33,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   private final long streamStartPositionUs;
   private final long streamOffsetUs;
   private final MuxerWrapper muxerWrapper;
-  private final Listener listener;
   private final @C.TrackType int trackType;
   private final @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener;
 
@@ -45,12 +44,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       long streamStartPositionUs,
       long streamOffsetUs,
       boolean flattenForSlowMotion,
-      MuxerWrapper muxerWrapper,
-      Listener listener) {
+      MuxerWrapper muxerWrapper) {
     this.streamStartPositionUs = streamStartPositionUs;
     this.streamOffsetUs = streamOffsetUs;
     this.muxerWrapper = muxerWrapper;
-    this.listener = listener;
     trackType = MimeTypes.getTrackType(inputFormat.sampleMimeType);
     sefVideoSlowMotionFlattener =
         flattenForSlowMotion && trackType == C.TRACK_TYPE_VIDEO
@@ -79,7 +76,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   @Override
   public void queueInputBuffer() throws TransformationException {
     DecoderInputBuffer inputBuffer = checkNotNull(this.inputBuffer);
-    listener.onInputBufferQueued(inputBuffer.timeUs - streamStartPositionUs);
     checkNotNull(inputBuffer.data);
     if (!shouldDropInputBuffer(inputBuffer)) {
       queueInputBufferInternal();

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java

@@ -20,6 +20,11 @@ import static com.google.android.exoplayer2.DefaultLoadControl.DEFAULT_BUFFER_FO
 import static com.google.android.exoplayer2.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_MS;
 import static com.google.android.exoplayer2.DefaultLoadControl.DEFAULT_MAX_BUFFER_MS;
 import static com.google.android.exoplayer2.DefaultLoadControl.DEFAULT_MIN_BUFFER_MS;
+import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
+import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_NO_TRANSFORMATION;
+import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
+import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+import static java.lang.Math.min;
 
 import android.content.Context;
 import android.os.Handler;
@@ -63,6 +68,8 @@ import com.google.android.exoplayer2.video.VideoRendererEventListener;
   private final MediaItem mediaItem;
   private final ExoPlayer player;
 
+  private @Transformer.ProgressState int progressState;
+
   public ExoPlayerAssetLoader(
       Context context,
       MediaItem mediaItem,
@@ -106,16 +113,29 @@ import com.google.android.exoplayer2.video.VideoRendererEventListener;
 
     player = playerBuilder.build();
     player.addListener(new PlayerListener(listener));
+
+    progressState = PROGRESS_STATE_NO_TRANSFORMATION;
   }
 
   public void start() {
     player.setMediaItem(mediaItem);
     player.prepare();
     player.play();
+    progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+  }
+
+  public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) {
+    if (progressState == PROGRESS_STATE_AVAILABLE) {
+      long durationMs = player.getDuration();
+      long positionMs = player.getCurrentPosition();
+      progressHolder.progress = min((int) (positionMs * 100 / durationMs), 99);
+    }
+    return progressState;
   }
 
   public void release() {
     player.release();
+    progressState = PROGRESS_STATE_NO_TRANSFORMATION;
   }
 
   private static final class RenderersFactoryImpl implements RenderersFactory {
@@ -167,7 +187,6 @@ import com.google.android.exoplayer2.video.VideoRendererEventListener;
   private final class PlayerListener implements Player.Listener {
 
     private final Listener listener;
-    private boolean hasSentDuration;
 
     public PlayerListener(Listener listener) {
       this.listener = listener;
@@ -175,14 +194,21 @@ import com.google.android.exoplayer2.video.VideoRendererEventListener;
 
     @Override
     public void onTimelineChanged(Timeline timeline, int reason) {
-      if (hasSentDuration) {
+      if (progressState != PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
         return;
       }
       Timeline.Window window = new Timeline.Window();
       timeline.getWindow(/* windowIndex= */ 0, window);
       if (!window.isPlaceholder) {
+        long durationUs = window.durationUs;
+        // Make progress permanently unavailable if the duration is unknown, so that it doesn't jump
+        // to a high value at the end of the transformation if the duration is set once the media is
+        // entirely loaded.
+        progressState =
+            durationUs <= 0 || durationUs == C.TIME_UNSET
+                ? PROGRESS_STATE_UNAVAILABLE
+                : PROGRESS_STATE_AVAILABLE;
         listener.onDurationUs(window.durationUs);
-        hasSentDuration = true;
       }
     }
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java

@@ -34,15 +34,13 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
       long streamOffsetUs,
       TransformationRequest transformationRequest,
       MuxerWrapper muxerWrapper,
-      Listener listener,
       FallbackListener fallbackListener) {
     super(
         format,
         streamStartPositionUs,
         streamOffsetUs,
         transformationRequest.flattenForSlowMotion,
-        muxerWrapper,
-        listener);
+        muxerWrapper);
     this.format = format;
     buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
     fallbackListener.onTransformationRequestFinalized(transformationRequest);

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java

@@ -40,25 +40,6 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
     void queueInputBuffer();
   }
 
-  /** A listener for the sample pipeline events. */
-  interface Listener {
-
-    /**
-     * Called when an input buffer is {@linkplain #queueInputBuffer() queued}.
-     *
-     * @param positionUs The position of the buffer queued from the stream start position, in
-     *     microseconds.
-     */
-    void onInputBufferQueued(long positionUs);
-
-    /**
-     * Called if an exception occurs in the sample pipeline.
-     *
-     * @param exception The {@link TransformationException} describing the exception.
-     */
-    void onTransformationError(TransformationException exception);
-  }
-
   /**
    * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample
    * data should be queued.

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java

@@ -774,7 +774,6 @@ public final class Transformer {
             encoderFactory,
             frameProcessorFactory,
             muxerFactory,
-            looper,
             transformerInternalListener,
             fallbackListener,
             debugViewProvider,

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java

@@ -17,11 +17,8 @@
 package com.google.android.exoplayer2.transformer;
 
 import static com.google.android.exoplayer2.transformer.TransformationException.ERROR_CODE_MUXING_FAILED;
-import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
-import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
-import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+import static com.google.android.exoplayer2.transformer.Transformer.PROGRESS_STATE_NO_TRANSFORMATION;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
-import static java.lang.Math.min;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
 import android.content.Context;
@@ -47,7 +44,6 @@ import com.google.android.exoplayer2.util.Effect;
 import com.google.android.exoplayer2.util.FrameProcessor;
 import com.google.android.exoplayer2.util.HandlerWrapper;
 import com.google.android.exoplayer2.util.MimeTypes;
-import com.google.android.exoplayer2.util.Util;
 import com.google.common.collect.ImmutableList;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
@@ -89,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private static final int MSG_QUEUE_INPUT = 3;
   private static final int MSG_DRAIN_PIPELINES = 4;
   private static final int MSG_END = 5;
+  private static final int MSG_UPDATE_PROGRESS = 6;
 
   private static final int DRAIN_PIPELINES_DELAY_MS = 50;
 
@@ -103,21 +100,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final Listener listener;
   private final DebugViewProvider debugViewProvider;
   private final Clock clock;
-  private final HandlerWrapper applicationHandler;
   private final HandlerThread internalHandlerThread;
   private final HandlerWrapper internalHandler;
   private final ExoPlayerAssetLoader exoPlayerAssetLoader;
   private final List<SamplePipeline> samplePipelines;
   private final ConditionVariable dequeueBufferConditionVariable;
   private final MuxerWrapper muxerWrapper;
-  private final ConditionVariable cancellingConditionVariable;
+  private final ConditionVariable transformerConditionVariable;
 
   @Nullable private DecoderInputBuffer pendingInputBuffer;
   private boolean isDrainingPipelines;
   private int silentSamplePipelineIndex;
   private @Transformer.ProgressState int progressState;
-  private long progressPositionMs;
-  private long durationUs;
   private @MonotonicNonNull RuntimeException cancelException;
 
   private volatile boolean released;
@@ -138,7 +132,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       Codec.EncoderFactory encoderFactory,
       FrameProcessor.Factory frameProcessorFactory,
       Muxer.Factory muxerFactory,
-      Looper applicationLooper,
       Listener listener,
       FallbackListener fallbackListener,
       DebugViewProvider debugViewProvider,
@@ -154,7 +147,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.listener = listener;
     this.debugViewProvider = debugViewProvider;
     this.clock = clock;
-    applicationHandler = clock.createHandler(applicationLooper, /* callback= */ null);
     internalHandlerThread = new HandlerThread("Transformer:Internal");
     internalHandlerThread.start();
     Looper internalLooper = internalHandlerThread.getLooper();
@@ -179,8 +171,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             outputParcelFileDescriptor,
             muxerFactory,
             /* errorConsumer= */ componentListener::onTransformationError);
-    cancellingConditionVariable = new ConditionVariable();
-    progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+    transformerConditionVariable = new ConditionVariable();
     // It's safe to use "this" because we don't send a message before exiting the constructor.
     @SuppressWarnings("nullness:methodref.receiver.bound")
     HandlerWrapper internalHandler =
@@ -193,9 +184,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) {
-    if (progressState == PROGRESS_STATE_AVAILABLE) {
-      progressHolder.progress = min((int) (progressPositionMs * 100 / Util.usToMs(durationUs)), 99);
+    if (released) {
+      return PROGRESS_STATE_NO_TRANSFORMATION;
     }
+    internalHandler.obtainMessage(MSG_UPDATE_PROGRESS, progressHolder).sendToTarget();
+    // TODO: figure out why calling clock.onThreadBlocked() here makes the tests fail.
+    transformerConditionVariable.blockUninterruptible();
+    transformerConditionVariable.close();
     return progressState;
   }
 
@@ -208,15 +203,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             MSG_END, END_REASON_CANCELLED, /* unused */ 0, /* transformationException */ null)
         .sendToTarget();
     clock.onThreadBlocked();
-    cancellingConditionVariable.blockUninterruptible();
+    transformerConditionVariable.blockUninterruptible();
+    transformerConditionVariable.close();
     if (cancelException != null) {
       throw cancelException;
     }
   }
 
   private boolean handleMessage(Message msg) {
-    // Handle end messages even if resources have been released to report release timeouts.
-    if (released && msg.what != MSG_END) {
+    // Some messages cannot be ignored when resources have been released. End messages must be
+    // handled to report release timeouts and to unblock the transformer condition variable in case
+    // of cancellation. Progress update messages must be handled to unblock the transformer
+    // condition variable.
+    if (released && msg.what != MSG_END && msg.what != MSG_UPDATE_PROGRESS) {
       return true;
     }
     try {
@@ -241,6 +240,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               /* endReason= */ msg.arg1,
               /* transformationException= */ (TransformationException) msg.obj);
           break;
+        case MSG_UPDATE_PROGRESS:
+          updateProgressInternal(/* progressHolder= */ (ProgressHolder) msg.obj);
+          break;
         default:
           return false;
       }
@@ -343,60 +345,53 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       internalHandler.post(internalHandlerThread::quitSafely);
     }
 
-    if (!forCancellation) {
-      TransformationException exception = transformationException;
-      if (exception == null) {
-        // We only report the exception caused by releasing the resources if there is no other
-        // exception. It is more intuitive to call the error callback only once and reporting the
-        // exception caused by releasing the resources can be confusing if it is a consequence of
-        // the first exception.
-        exception = releaseTransformationException;
-      }
+    if (forCancellation) {
+      transformerConditionVariable.open();
+      return;
+    }
 
-      if (exception != null) {
-        listener.onTransformationError(exception);
-      } else {
-        listener.onTransformationCompleted(checkNotNull(transformationResult));
-      }
+    TransformationException exception = transformationException;
+    if (exception == null) {
+      // We only report the exception caused by releasing the resources if there is no other
+      // exception. It is more intuitive to call the error callback only once and reporting the
+      // exception caused by releasing the resources can be confusing if it is a consequence of the
+      // first exception.
+      exception = releaseTransformationException;
     }
 
-    cancellingConditionVariable.open();
+    if (exception != null) {
+      listener.onTransformationError(exception);
+    } else {
+      listener.onTransformationCompleted(checkNotNull(transformationResult));
+    }
   }
 
-  private class ComponentListener
-      implements ExoPlayerAssetLoader.Listener, SamplePipeline.Listener {
+  private void updateProgressInternal(ProgressHolder progressHolder) {
+    progressState = exoPlayerAssetLoader.getProgress(progressHolder);
+    transformerConditionVariable.open();
+  }
 
-    private static final long MIN_DURATION_BETWEEN_PROGRESS_UPDATES_MS = 100;
+  private class ComponentListener implements ExoPlayerAssetLoader.Listener {
 
     private final MediaItem mediaItem;
     private final FallbackListener fallbackListener;
 
+    private long durationUs;
     private int tracksAddedCount;
-    private long lastProgressUpdateMs;
-    private long lastProgressPositionMs;
 
     private volatile boolean trackRegistered;
 
     public ComponentListener(MediaItem mediaItem, FallbackListener fallbackListener) {
       this.mediaItem = mediaItem;
       this.fallbackListener = fallbackListener;
+      durationUs = C.TIME_UNSET;
     }
 
     // ExoPlayerAssetLoader.Listener implementation.
 
     @Override
     public void onDurationUs(long durationUs) {
-      applicationHandler.post(
-          () -> {
-            // Make progress permanently unavailable if the duration is unknown, so that it doesn't
-            // jump to a high value at the end of the transformation if the duration is set once the
-            // media is entirely loaded.
-            progressState =
-                durationUs <= 0 || durationUs == C.TIME_UNSET
-                    ? PROGRESS_STATE_UNAVAILABLE
-                    : PROGRESS_STATE_AVAILABLE;
-            TransformerInternal.this.durationUs = durationUs;
-          });
+      this.durationUs = durationUs;
     }
 
     @Override
@@ -462,22 +457,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       onTransformationError(transformationException);
     }
 
-    // SamplePipeline.Listener implementation.
-
-    @Override
-    public void onInputBufferQueued(long positionUs) {
-      long positionMs = Util.usToMs(positionUs);
-      long elapsedTimeMs = clock.elapsedRealtime();
-      if (elapsedTimeMs > lastProgressUpdateMs + MIN_DURATION_BETWEEN_PROGRESS_UPDATES_MS
-          && positionMs > lastProgressPositionMs) {
-        lastProgressUpdateMs = elapsedTimeMs;
-        // Store positionMs in a variable to make sure the thread reads the latest value.
-        lastProgressPositionMs = positionMs;
-        applicationHandler.post(() -> progressPositionMs = positionMs);
-      }
-    }
-
-    @Override
     public void onTransformationError(TransformationException transformationException) {
       internalHandler
           .obtainMessage(MSG_END, END_REASON_ERROR, /* unused */ 0, transformationException)
@@ -497,7 +476,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             forceSilentAudio ? durationUs : C.TIME_UNSET,
             encoderFactory,
             muxerWrapper,
-            /* listener= */ this,
             fallbackListener);
       } else if (MimeTypes.isVideo(inputFormat.sampleMimeType)
           && shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs)) {
@@ -512,7 +490,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             decoderFactory,
             encoderFactory,
             muxerWrapper,
-            /* listener= */ this,
+            /* errorConsumer= */ this::onTransformationError,
             fallbackListener,
             debugViewProvider);
       } else {
@@ -522,7 +500,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             streamOffsetUs,
             transformationRequest,
             muxerWrapper,
-            /* listener= */ this,
             fallbackListener);
       }
     }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.effect.Presentation;
 import com.google.android.exoplayer2.effect.ScaleToFitTransformation;
+import com.google.android.exoplayer2.util.Consumer;
 import com.google.android.exoplayer2.util.DebugViewProvider;
 import com.google.android.exoplayer2.util.Effect;
 import com.google.android.exoplayer2.util.FrameInfo;
@@ -82,7 +83,7 @@ import org.checkerframework.dataflow.qual.Pure;
       Codec.DecoderFactory decoderFactory,
       Codec.EncoderFactory encoderFactory,
       MuxerWrapper muxerWrapper,
-      Listener listener,
+      Consumer<TransformationException> errorConsumer,
       FallbackListener fallbackListener,
       DebugViewProvider debugViewProvider)
       throws TransformationException {
@@ -91,8 +92,7 @@ import org.checkerframework.dataflow.qual.Pure;
         streamStartPositionUs,
         streamOffsetUs,
         transformationRequest.flattenForSlowMotion,
-        muxerWrapper,
-        listener);
+        muxerWrapper);
 
     if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) {
       if (transformationRequest.hdrMode
@@ -179,7 +179,7 @@ import org.checkerframework.dataflow.qual.Pure;
                     checkNotNull(frameProcessor)
                         .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
                   } catch (TransformationException exception) {
-                    listener.onTransformationError(exception);
+                    errorConsumer.accept(exception);
                   }
                 }
 
@@ -191,7 +191,7 @@ import org.checkerframework.dataflow.qual.Pure;
 
                 @Override
                 public void onFrameProcessingError(FrameProcessingException exception) {
-                  listener.onTransformationError(
+                  errorConsumer.accept(
                       TransformationException.createForFrameProcessingException(
                           exception, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED));
                 }
@@ -203,7 +203,7 @@ import org.checkerframework.dataflow.qual.Pure;
                   try {
                     encoderWrapper.signalEndOfInputStream();
                   } catch (TransformationException exception) {
-                    listener.onTransformationError(exception);
+                    errorConsumer.accept(exception);
                   }
                 }
               });