Expose experimental offload scheduling

Add a new scheduling mode that stops ExoPlayer main loop
when the audio offload buffer is full and resume it when
it has been partially played.

This mode needs to be enabled and dissabled manually by the app
for now.

#exo-offload

PiperOrigin-RevId: 315860373

RELEASENOTES.md

@@ -158,6 +158,7 @@
     *   No longer use a `MediaCodec` in audio passthrough mode.
     *   Check `DefaultAudioSink` supports passthrough, in addition to checking
         the `AudioCapabilities`
+    *   Add an experimental scheduling mode to save power in offload.
         ([#7404](https://github.com/google/ExoPlayer/issues/7404)).
     *   Adjust input timestamps in `MediaCodecRenderer` to account for the
         Codec2 MP3 decoder having lower timestamps on the output side.

library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java

@@ -219,12 +219,20 @@ public class DefaultRenderersFactory implements RenderersFactory {
   }
 
   /**
-   * Sets whether audio should be played using the offload path. Audio offload disables audio
-   * processors (for example speed adjustment).
+   * Sets whether audio should be played using the offload path.
+   *
+   * <p>Audio offload disables ExoPlayer audio processing, but significantly reduces the energy
+   * consumption of the playback when {@link
+   * ExoPlayer#experimental_enableOffloadScheduling(boolean)} is enabled.
+   *
+   * <p>Most Android devices can only support one offload {@link android.media.AudioTrack} at a time
+   * and can invalidate it at any time. Thus an app can never be guaranteed that it will be able to
+   * play in offload.
    *
    * <p>The default value is {@code false}.
    *
-   * @param enableOffload If audio offload should be used.
+   * @param enableOffload Whether to enable use of audio offload for supported formats, if
+   *     available.
    * @return This factory, for convenience.
    */
   public DefaultRenderersFactory setEnableAudioOffload(boolean enableOffload) {
@@ -423,7 +431,8 @@ public class DefaultRenderersFactory implements RenderersFactory {
    *     before output. May be empty.
    * @param eventHandler A handler to use when invoking event listeners and outputs.
    * @param eventListener An event listener.
-   * @param enableOffload If the renderer should use audio offload for all supported formats.
+   * @param enableOffload Whether to enable use of audio offload for supported formats, if
+   *     available.
    * @param out An array to which the built renderers should be appended.
    */
   protected void buildAudioRenderers(

library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java

@@ -20,6 +20,8 @@ import android.os.Looper;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import com.google.android.exoplayer2.analytics.AnalyticsCollector;
+import com.google.android.exoplayer2.audio.AudioCapabilities;
+import com.google.android.exoplayer2.audio.DefaultAudioSink;
 import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer;
 import com.google.android.exoplayer2.metadata.MetadataRenderer;
 import com.google.android.exoplayer2.source.ClippingMediaSource;
@@ -597,4 +599,39 @@ public interface ExoPlayer extends Player {
    * @see #setPauseAtEndOfMediaItems(boolean)
    */
   boolean getPauseAtEndOfMediaItems();
+
+  /**
+   * Enables audio offload scheduling, which runs ExoPlayer's main loop as rarely as possible when
+   * playing an audio stream using audio offload.
+   *
+   * <p>Only use this scheduling mode if the player is not displaying anything to the user. For
+   * example when the application is in the background, or the screen is off. The player state
+   * (including position) is rarely updated (between 10s and 1min).
+   *
+   * <p>While offload scheduling is enabled, player events may be delivered severely delayed and
+   * apps should not interact with the player. When returning to the foreground, disable offload
+   * scheduling before interacting with the player
+   *
+   * <p>This mode should save significant power when the phone is playing offload audio with the
+   * screen off.
+   *
+   * <p>This mode only has an effect when playing an audio track in offload mode, which requires all
+   * the following:
+   *
+   * <ul>
+   *   <li>audio offload rendering is enabled in {@link
+   *       DefaultRenderersFactory#setEnableAudioOffload} or the equivalent option passed to {@link
+   *       com.google.android.exoplayer2.audio.DefaultAudioSink#DefaultAudioSink(AudioCapabilities,
+   *       DefaultAudioSink.AudioProcessorChain, boolean, boolean)}.
+   *   <li>an audio track is playing in a format which the device supports offloading (for example
+   *       MP3 or AAC).
+   *   <li>The {@link com.google.android.exoplayer2.audio.AudioSink} is playing with an offload
+   *       {@link android.media.AudioTrack}.
+   * </ul>
+   *
+   * <p>This method is experimental, and will be renamed or removed in a future release.
+   *
+   * @param enableOffloadScheduling Whether to enable offload scheduling.
+   */
+  void experimental_enableOffloadScheduling(boolean enableOffloadScheduling);
 }

library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java

@@ -203,6 +203,11 @@ import java.util.concurrent.TimeoutException;
   }
 
   @Override
+  public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) {
+    internalPlayer.experimental_enableOffloadScheduling(enableOffloadScheduling);
+  }
+
+  @Override
   @Nullable
   public AudioComponent getAudioComponent() {
     return null;

library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java

@@ -94,6 +94,15 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
   private static final int ACTIVE_INTERVAL_MS = 10;
   private static final int IDLE_INTERVAL_MS = 1000;
+  /**
+   * Duration under which pausing the main DO_SOME_WORK loop is not expected to yield significant
+   * power saving.
+   *
+   * <p>This value is probably too high, power measurements are needed adjust it, but as renderer
+   * sleep is currently only implemented for audio offload, which uses buffer much bigger than 2s,
+   * this does not matter for now.
+   */
+  private static final long MIN_RENDERER_SLEEP_DURATION_MS = 2000;
 
   private final Renderer[] renderers;
   private final RendererCapabilities[] rendererCapabilities;
@@ -127,6 +136,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
   @Player.RepeatMode private int repeatMode;
   private boolean shuffleModeEnabled;
   private boolean foregroundMode;
+  private boolean requestForRendererSleep;
+  private boolean offloadSchedulingEnabled;
 
   private int enabledRendererCount;
   @Nullable private SeekPosition pendingInitialSeekPosition;
@@ -199,6 +210,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
     throwWhenStuckBuffering = true;
   }
 
+  public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) {
+    offloadSchedulingEnabled = enableOffloadScheduling;
+    if (!enableOffloadScheduling) {
+      handler.sendEmptyMessage(MSG_DO_SOME_WORK);
+    }
+  }
+
   public void prepare() {
     handler.obtainMessage(MSG_PREPARE).sendToTarget();
   }
@@ -885,12 +903,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
     if ((shouldPlayWhenReady() && playbackInfo.playbackState == Player.STATE_READY)
         || playbackInfo.playbackState == Player.STATE_BUFFERING) {
-      scheduleNextWork(operationStartTimeMs, ACTIVE_INTERVAL_MS);
+      maybeScheduleWakeup(operationStartTimeMs, ACTIVE_INTERVAL_MS);
     } else if (enabledRendererCount != 0 && playbackInfo.playbackState != Player.STATE_ENDED) {
       scheduleNextWork(operationStartTimeMs, IDLE_INTERVAL_MS);
     } else {
       handler.removeMessages(MSG_DO_SOME_WORK);
     }
+    requestForRendererSleep = false; // A sleep request is only valid for the current doSomeWork.
 
     TraceUtil.endSection();
   }
@@ -900,6 +919,14 @@ import java.util.concurrent.atomic.AtomicBoolean;
     handler.sendEmptyMessageAtTime(MSG_DO_SOME_WORK, thisOperationStartTimeMs + intervalMs);
   }
 
+  private void maybeScheduleWakeup(long operationStartTimeMs, long intervalMs) {
+    if (offloadSchedulingEnabled && requestForRendererSleep) {
+      return;
+    }
+
+    scheduleNextWork(operationStartTimeMs, intervalMs);
+  }
+
   private void seekToInternal(SeekPosition seekPosition) throws ExoPlaybackException {
     playbackInfoUpdate.incrementPendingOperationAcks(/* operationAcks= */ 1);
 
@@ -2068,6 +2095,24 @@ import java.util.concurrent.atomic.AtomicBoolean;
         joining,
         mayRenderStartOfStream,
         periodHolder.getRendererOffset());
+
+    renderer.handleMessage(
+        Renderer.MSG_SET_WAKEUP_LISTENER,
+        new Renderer.WakeupListener() {
+          @Override
+          public void onSleep(long wakeupDeadlineMs) {
+            // Do not sleep if the expected sleep time is not long enough to save significant power.
+            if (wakeupDeadlineMs >= MIN_RENDERER_SLEEP_DURATION_MS) {
+              requestForRendererSleep = true;
+            }
+          }
+
+          @Override
+          public void onWakeup() {
+            handler.sendEmptyMessage(MSG_DO_SOME_WORK);
+          }
+        });
+
     mediaClock.onRendererEnabled(renderer);
     // Start the renderer if playing.
     if (playing) {

library/core/src/main/java/com/google/android/exoplayer2/Renderer.java

@@ -47,6 +47,30 @@ import java.lang.annotation.RetentionPolicy;
 public interface Renderer extends PlayerMessage.Target {
 
   /**
+   * Some renderers can signal when {@link #render(long, long)} should be called.
+   *
+   * <p>That allows the player to sleep until the next wakeup, instead of calling {@link
+   * #render(long, long)} in a tight loop. The aim of this interrupt based scheduling is to save
+   * power.
+   */
+  interface WakeupListener {
+
+    /**
+     * The renderer no longer needs to render until the next wakeup.
+     *
+     * @param wakeupDeadlineMs Maximum time in milliseconds until {@link #onWakeup()} will be
+     *     called.
+     */
+    void onSleep(long wakeupDeadlineMs);
+
+    /**
+     * The renderer needs to render some frames. The client should call {@link #render(long, long)}
+     * at its earliest convenience.
+     */
+    void onWakeup();
+  }
+
+  /**
    * The type of a message that can be passed to a video renderer via {@link
    * ExoPlayer#createMessage(Target)}. The message payload should be the target {@link Surface}, or
    * null.
@@ -138,6 +162,14 @@ public interface Renderer extends PlayerMessage.Target {
    */
   int MSG_SET_AUDIO_SESSION_ID = 102;
   /**
+   * A type of a message that can be passed to a {@link Renderer} via {@link
+   * ExoPlayer#createMessage(Target)}, to inform the renderer that it can schedule waking up another
+   * component.
+   *
+   * <p>The message payload must be a {@link WakeupListener} instance.
+   */
+  int MSG_SET_WAKEUP_LISTENER = 103;
+  /**
    * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
    * renderers. These custom constants must be greater than or equal to this value.
    */

library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java

@@ -634,6 +634,11 @@ public class SimpleExoPlayer extends BasePlayer
   }
 
   @Override
+  public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) {
+    player.experimental_enableOffloadScheduling(enableOffloadScheduling);
+  }
+
+  @Override
   @Nullable
   public AudioComponent getAudioComponent() {
     return this;

library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java

@@ -90,6 +90,17 @@ public interface AudioSink {
      * @param skipSilenceEnabled Whether skipping silences is enabled.
      */
     void onSkipSilenceEnabledChanged(boolean skipSilenceEnabled);
+
+    /** Called when the offload buffer has been partially emptied. */
+    default void onOffloadBufferEmptying() {}
+
+    /**
+     * Called when the offload buffer has been filled completely.
+     *
+     * @param bufferEmptyingDeadlineMs Maximum time in milliseconds until {@link
+     *     #onOffloadBufferEmptying()} will be called.
+     */
+    default void onOffloadBufferFull(long bufferEmptyingDeadlineMs) {}
   }
 
   /**

library/core/src/main/java/com/google/android/exoplayer2/audio/AudioTrackPositionTracker.java

@@ -335,6 +335,11 @@ import java.lang.reflect.Method;
     return bufferSize - bytesPending;
   }
 
+  /** Returns the duration of audio that is buffered but unplayed. */
+  public long getPendingBufferDurationMs(long writtenFrames) {
+    return C.usToMs(framesToDurationUs(writtenFrames - getPlaybackHeadPosition()));
+  }
+
   /** Returns whether the track is in an invalid state and must be recreated. */
   public boolean isStalled(long writtenFrames) {
     return forceResetWorkaroundTimeMs != C.TIME_UNSET

library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java

@@ -20,6 +20,7 @@ import android.media.AudioFormat;
 import android.media.AudioManager;
 import android.media.AudioTrack;
 import android.os.ConditionVariable;
+import android.os.Handler;
 import android.os.SystemClock;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
@@ -35,6 +36,7 @@ import java.nio.ByteOrder;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
@@ -274,6 +276,7 @@ public final class DefaultAudioSink implements AudioSink {
   private final AudioTrackPositionTracker audioTrackPositionTracker;
   private final ArrayDeque<MediaPositionParameters> mediaPositionParametersCheckpoints;
   private final boolean enableOffload;
+  @Nullable private final StreamEventCallback offloadStreamEventCallback;
 
   @Nullable private Listener listener;
   /** Used to keep the audio session active on pre-V21 builds (see {@link #initialize(long)}). */
@@ -366,7 +369,10 @@ public final class DefaultAudioSink implements AudioSink {
    *     be available when float output is in use.
    * @param enableOffload Whether audio offloading is enabled. If an audio format can be both played
    *     with offload and encoded audio passthrough, it will be played in offload. Audio offload is
-   *     supported starting with API 29 ({@link android.os.Build.VERSION_CODES#Q}).
+   *     supported starting with API 29 ({@link android.os.Build.VERSION_CODES#Q}). Most Android
+   *     devices can only support one offload {@link android.media.AudioTrack} at a time and can
+   *     invalidate it at any time. Thus an app can never be guaranteed that it will be able to play
+   *     in offload.
    */
   public DefaultAudioSink(
       @Nullable AudioCapabilities audioCapabilities,
@@ -404,6 +410,7 @@ public final class DefaultAudioSink implements AudioSink {
     activeAudioProcessors = new AudioProcessor[0];
     outputBuffers = new ByteBuffer[0];
     mediaPositionParametersCheckpoints = new ArrayDeque<>();
+    offloadStreamEventCallback = Util.SDK_INT >= 29 ? new StreamEventCallback() : null;
   }
 
   // AudioSink implementation.
@@ -563,6 +570,10 @@ public final class DefaultAudioSink implements AudioSink {
     audioTrack =
         Assertions.checkNotNull(configuration)
             .buildAudioTrack(tunneling, audioAttributes, audioSessionId);
+    if (isOffloadedPlayback(audioTrack)) {
+      // Receive stream event callbacks on the current (playback) thread.
+      Assertions.checkNotNull(offloadStreamEventCallback).register(audioTrack);
+    }
     int audioSessionId = audioTrack.getAudioSessionId();
     if (enablePreV21AudioSessionWorkaround) {
       if (Util.SDK_INT < 21) {
@@ -822,6 +833,15 @@ public final class DefaultAudioSink implements AudioSink {
       throw new WriteException(bytesWritten);
     }
 
+    if (isOffloadedPlayback(audioTrack)
+        && playing
+        && listener != null
+        && bytesWritten < bytesRemaining) {
+      long pendingDurationMs =
+          audioTrackPositionTracker.getPendingBufferDurationMs(writtenEncodedFrames);
+      listener.onOffloadBufferFull(pendingDurationMs);
+    }
+
     if (configuration.isInputPcm) {
       writtenPcmBytes += bytesWritten;
     }
@@ -1040,6 +1060,9 @@ public final class DefaultAudioSink implements AudioSink {
       if (audioTrackPositionTracker.isPlaying()) {
         audioTrack.pause();
       }
+      if (isOffloadedPlayback(audioTrack)) {
+        offloadStreamEventCallback.unregister(audioTrack);
+      }
       // AudioTrack.release can take some time, so we call it on a background thread.
       final AudioTrack toRelease = audioTrack;
       audioTrack = null;
@@ -1229,6 +1252,39 @@ public final class DefaultAudioSink implements AudioSink {
         audioFormat, audioAttributes.getAudioAttributesV21());
   }
 
+  @EnsuresNonNullIf(
+      result = true,
+      expression = {"offloadStreamEventCallback"})
+  private static boolean isOffloadedPlayback(AudioTrack audioTrack) {
+    return Util.SDK_INT >= 29 && audioTrack.isOffloadedPlayback();
+  }
+
+  @RequiresApi(29)
+  private final class StreamEventCallback extends AudioTrack.StreamEventCallback {
+    private final Handler handler;
+
+    public StreamEventCallback() {
+      handler = new Handler();
+    }
+
+    @Override
+    public void onDataRequest(AudioTrack track, int size) {
+      Assertions.checkState(track == DefaultAudioSink.this.audioTrack);
+      if (listener != null) {
+        listener.onOffloadBufferEmptying();
+      }
+    }
+
+    public void register(AudioTrack audioTrack) {
+      audioTrack.registerStreamEventCallback(handler::post, this);
+    }
+
+    public void unregister(AudioTrack audioTrack) {
+      audioTrack.unregisterStreamEventCallback(this);
+      handler.removeCallbacksAndMessages(/* token= */ null);
+    }
+  }
+
   private static AudioTrack initializeKeepSessionIdAudioTrack(int audioSessionId) {
     int sampleRate = 4000; // Equal to private AudioTrack.MIN_SAMPLE_RATE.
     int channelConfig = AudioFormat.CHANNEL_OUT_MONO;

library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java

@@ -91,6 +91,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   private boolean allowFirstBufferPositionDiscontinuity;
   private boolean allowPositionDiscontinuity;
 
+  @Nullable private WakeupListener wakeupListener;
+
   /**
    * @param context A context.
    * @param mediaCodecSelector A decoder selector.
@@ -695,6 +697,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       case MSG_SET_AUDIO_SESSION_ID:
         audioSink.setAudioSessionId((Integer) message);
         break;
+      case MSG_SET_WAKEUP_LISTENER:
+        this.wakeupListener = (WakeupListener) message;
+        break;
       default:
         super.handleMessage(messageType, message);
         break;
@@ -874,5 +879,19 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       eventDispatcher.skipSilenceEnabledChanged(skipSilenceEnabled);
       onAudioTrackSkipSilenceEnabledChanged(skipSilenceEnabled);
     }
+
+    @Override
+    public void onOffloadBufferEmptying() {
+      if (wakeupListener != null) {
+        wakeupListener.onWakeup();
+      }
+    }
+
+    @Override
+    public void onOffloadBufferFull(long bufferEmptyingDeadlineMs) {
+      if (wakeupListener != null) {
+        wakeupListener.onSleep(bufferEmptyingDeadlineMs);
+      }
+    }
   }
 }

testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java

@@ -465,4 +465,9 @@ public abstract class StubExoPlayer extends BasePlayer implements ExoPlayer {
   public boolean getPauseAtEndOfMediaItems() {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public void experimental_enableOffloadScheduling(boolean enableOffloadScheduling) {
+    throw new UnsupportedOperationException();
+  }
 }